From 0a6d1226234ca78eec2d97c621eec4ad79aa369d Mon Sep 17 00:00:00 2001
From: Derek Higgins <derekh@redhat.com>
Date: Wed, 13 Aug 2025 14:19:52 +0100
Subject: [PATCH] ci: integrate vLLM inference tests with GitHub Actions
 workflows

Add vLLM provider support to integration test CI workflows alongside
existing Ollama support. Configure provider-specific test execution
where vLLM runs only inference specific tests (excluding vision tests) while
Ollama continues to run the full test suite.

This enables comprehensive CI testing of both inference providers but
keeps the vLLM footprint small, this can be expanded later if it proves
to not be too disruptive.

Also updated test skips that were marked with "inline::vllm", this
should be "remote::vllm". This causes some failing log probs tests
to be skipped and should be revisted.

Signed-off-by: Derek Higgins <derekh@redhat.com>
---
 .github/actions/run-and-record-tests/action.yml | 5 ++++-
 .github/workflows/integration-tests.yml         | 1 -
 tests/integration/ci_matrix.json                | 3 ++-
 tests/integration/suites.py                     | 5 +++++
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/actions/run-and-record-tests/action.yml b/.github/actions/run-and-record-tests/action.yml
index ec4d7f977..d44cba4ee 100644
--- a/.github/actions/run-and-record-tests/action.yml
+++ b/.github/actions/run-and-record-tests/action.yml
@@ -72,7 +72,8 @@ runs:
           echo "New recordings detected, committing and pushing"
           git add tests/integration/
 
-          git commit -m "Recordings update from CI (suite: ${{ inputs.suite }})"
+          git commit -m "Recordings update from CI (setup: ${{ inputs.setup }}, suite: ${{ inputs.suite }})"
+
           git fetch origin ${{ github.ref_name }}
           git rebase origin/${{ github.ref_name }}
           echo "Rebased successfully"
@@ -88,6 +89,8 @@ runs:
       run: |
         # Ollama logs (if ollama container exists)
         sudo docker logs ollama > ollama-${{ inputs.inference-mode }}.log 2>&1 || true
+        # vllm logs (if vllm container exists)
+        sudo docker logs vllm > vllm-${{ inputs.inference-mode }}.log 2>&1 || true
         # Note: distro container logs are now dumped in integration-tests.sh before container is removed
 
     - name: Upload logs
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 41822fa79..2c797e906 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -27,7 +27,6 @@ on:
   schedule:
     # If changing the cron schedule, update the provider in the test-matrix job
     - cron: '0 0 * * *'  # (test latest client) Daily at 12 AM UTC
-    - cron: '1 0 * * 0'  # (test vllm) Weekly on Sunday at 1 AM UTC
   workflow_dispatch:
     inputs:
       test-all-client-versions:
diff --git a/tests/integration/ci_matrix.json b/tests/integration/ci_matrix.json
index 314070eab..858176dff 100644
--- a/tests/integration/ci_matrix.json
+++ b/tests/integration/ci_matrix.json
@@ -2,7 +2,8 @@
   "default": [
     {"suite": "base", "setup": "ollama"},
     {"suite": "vision", "setup": "ollama-vision"},
-    {"suite": "responses", "setup": "gpt"}
+    {"suite": "responses", "setup": "gpt"},
+    {"suite": "base-vllm-subset", "setup": "vllm"}
   ],
   "schedules": {
     "1 0 * * 0": [
diff --git a/tests/integration/suites.py b/tests/integration/suites.py
index 629f87775..0cec66afe 100644
--- a/tests/integration/suites.py
+++ b/tests/integration/suites.py
@@ -169,6 +169,11 @@ SUITE_DEFINITIONS: dict[str, Suite] = {
         roots=base_roots,
         default_setup="ollama",
     ),
+    "base-vllm-subset": Suite(
+        name="base-vllm-subset",
+        roots=["tests/integration/inference"],
+        default_setup="vllm",
+    ),
     "responses": Suite(
         name="responses",
         roots=["tests/integration/responses"],