feat(tests): add TypeScript client integration test support

Integration tests now support replaying TypeScript SDK tests alongside Python tests when running against server-mode stacks. This enables cross-language validation of API contracts and ensures the TypeScript client properly handles recorded responses.

The implementation adds a new `RUN_CLIENT_TS_TESTS` environment variable that triggers TypeScript test execution after successful Python runs. A mapping file (`suites.json`) defines which TypeScript test files correspond to each Python test suite/setup combination. The script automatically installs npm dependencies, forwards server configuration (base URL and model defaults from setup definitions), and executes matching TypeScript tests using Jest.

CI integration is enabled for server-based test jobs, and the feature can be exercised locally with commands like:

\`\`\`bash
RUN_CLIENT_TS_TESTS=1 scripts/integration-tests.sh --stack-config server:ci-tests --suite responses --setup gpt
\`\`\`

The TypeScript tests reuse existing replay fixtures through the forwarded \`TEST_API_BASE_URL\`, avoiding the need for duplicate response recordings.

tests/integration/README.md

@@ -211,3 +211,13 @@ def test_asymmetric_embeddings(llama_stack_client, embedding_model_id):
 
     assert query_response.embeddings is not None
 ```
+
+## TypeScript Client Replays
+
+Setting `RUN_CLIENT_TS_TESTS=1` when running `scripts/integration-tests.sh` against a `server:<config>` stack will replay the matching TypeScript SDK suites from `tests/integration/client-typescript/` immediately after the Python run. The mapping between suites/setups and `.test.ts` files lives in `tests/integration/client-typescript/suites.json`. This mode is enabled in CI for the `server` client jobs, and you can exercise it locally with commands such as:
+
+```bash
+RUN_CLIENT_TS_TESTS=1 scripts/integration-tests.sh --stack-config server:ci-tests --suite responses --setup gpt
+```
+
+The script installs the npm project on demand and forwards the server's `TEST_API_BASE_URL` + model defaults so the TypeScript tests can reuse the existing replay fixtures.

tests/integration/client-typescript/__tests__/inference.test.ts

@@ -0,0 +1,104 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+/**
+ * Integration tests for Inference API (Chat Completions).
+ * Ported from: llama-stack/tests/integration/inference/test_openai_completion.py
+ *
+ * IMPORTANT: Test cases must match EXACTLY with Python tests to use recorded API responses.
+ */
+
+import { createTestClient, requireTextModel } from '../setup';
+
+describe('Inference API - Chat Completions', () => {
+  // Test cases matching llama-stack/tests/integration/test_cases/inference/chat_completion.json
+  const chatCompletionTestCases = [
+    {
+      id: 'non_streaming_01',
+      question: 'Which planet do humans live on?',
+      expected: 'earth',
+      testId:
+        'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_non_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:non_streaming_01]',
+    },
+    {
+      id: 'non_streaming_02',
+      question: 'Which planet has rings around it with a name starting with letter S?',
+      expected: 'saturn',
+      testId:
+        'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_non_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:non_streaming_02]',
+    },
+  ];
+
+  const streamingTestCases = [
+    {
+      id: 'streaming_01',
+      question: "What's the name of the Sun in latin?",
+      expected: 'sol',
+      testId:
+        'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:streaming_01]',
+    },
+    {
+      id: 'streaming_02',
+      question: 'What is the name of the US captial?',
+      expected: 'washington',
+      testId:
+        'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:streaming_02]',
+    },
+  ];
+
+  test.each(chatCompletionTestCases)(
+    'chat completion non-streaming: $id',
+    async ({ question, expected, testId }) => {
+      const client = createTestClient(testId);
+      const textModel = requireTextModel();
+
+      const response = await client.chat.completions.create({
+        model: textModel,
+        messages: [
+          {
+            role: 'user',
+            content: question,
+          },
+        ],
+        stream: false,
+      });
+
+      // Non-streaming responses have choices with message property
+      const choice = response.choices[0];
+      expect(choice).toBeDefined();
+      if (!choice || !('message' in choice)) {
+        throw new Error('Expected non-streaming response with message');
+      }
+      const content = choice.message.content;
+      expect(content).toBeDefined();
+      const messageContent = typeof content === 'string' ? content.toLowerCase().trim() : '';
+      expect(messageContent.length).toBeGreaterThan(0);
+      expect(messageContent).toContain(expected.toLowerCase());
+    },
+  );
+
+  test.each(streamingTestCases)('chat completion streaming: $id', async ({ question, expected, testId }) => {
+    const client = createTestClient(testId);
+    const textModel = requireTextModel();
+
+    const stream = await client.chat.completions.create({
+      model: textModel,
+      messages: [{ role: 'user', content: question }],
+      stream: true,
+    });
+
+    const streamedContent: string[] = [];
+    for await (const chunk of stream) {
+      if (chunk.choices && chunk.choices.length > 0 && chunk.choices[0]?.delta?.content) {
+        streamedContent.push(chunk.choices[0].delta.content);
+      }
+    }
+
+    expect(streamedContent.length).toBeGreaterThan(0);
+    const fullContent = streamedContent.join('').toLowerCase().trim();
+    expect(fullContent).toContain(expected.toLowerCase());
+  });
+});

tests/integration/client-typescript/__tests__/responses.test.ts

@@ -0,0 +1,132 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+/**
+ * Integration tests for Responses API.
+ * Ported from: llama-stack/tests/integration/responses/test_basic_responses.py
+ *
+ * IMPORTANT: Test cases and IDs must match EXACTLY with Python tests to use recorded API responses.
+ */
+
+import { createTestClient, requireTextModel } from '../setup';
+
+describe('Responses API - Basic', () => {
+  // Test cases matching llama-stack/tests/integration/responses/fixtures/test_cases.py
+  const basicTestCases = [
+    {
+      id: 'earth',
+      input: 'Which planet do humans live on?',
+      expected: 'earth',
+      // Use client_with_models fixture to match non-streaming recordings
+      testId:
+        'tests/integration/responses/test_basic_responses.py::test_response_non_streaming_basic[client_with_models-txt=openai/gpt-4o-earth]',
+    },
+    {
+      id: 'saturn',
+      input: 'Which planet has rings around it with a name starting with letter S?',
+      expected: 'saturn',
+      testId:
+        'tests/integration/responses/test_basic_responses.py::test_response_non_streaming_basic[client_with_models-txt=openai/gpt-4o-saturn]',
+    },
+  ];
+
+  test.each(basicTestCases)('non-streaming basic response: $id', async ({ input, expected, testId }) => {
+    // Create client with test_id for all requests
+    const client = createTestClient(testId);
+    const textModel = requireTextModel();
+
+    // Create a response
+    const response = await client.responses.create({
+      model: textModel,
+      input,
+      stream: false,
+    });
+
+    // Verify response has content
+    const outputText = response.output_text.toLowerCase().trim();
+    expect(outputText.length).toBeGreaterThan(0);
+    expect(outputText).toContain(expected.toLowerCase());
+
+    // Verify usage is reported
+    expect(response.usage).toBeDefined();
+    expect(response.usage!.input_tokens).toBeGreaterThan(0);
+    expect(response.usage!.output_tokens).toBeGreaterThan(0);
+    expect(response.usage!.total_tokens).toBe(response.usage!.input_tokens + response.usage!.output_tokens);
+
+    // Verify stored response matches
+    const retrievedResponse = await client.responses.retrieve(response.id);
+    expect(retrievedResponse.output_text).toBe(response.output_text);
+
+    // Test follow-up with previous_response_id
+    const nextResponse = await client.responses.create({
+      model: textModel,
+      input: 'Repeat your previous response in all caps.',
+      previous_response_id: response.id,
+    });
+    const nextOutputText = nextResponse.output_text.trim();
+    expect(nextOutputText).toContain(expected.toUpperCase());
+  });
+
+  test.each(basicTestCases)('streaming basic response: $id', async ({ input, expected, testId }) => {
+    // Modify test_id for streaming variant
+    const streamingTestId = testId.replace(
+      'test_response_non_streaming_basic',
+      'test_response_streaming_basic',
+    );
+    const client = createTestClient(streamingTestId);
+    const textModel = requireTextModel();
+
+    // Create a streaming response
+    const stream = await client.responses.create({
+      model: textModel,
+      input,
+      stream: true,
+    });
+
+    const events: any[] = [];
+    let responseId = '';
+
+    for await (const chunk of stream) {
+      events.push(chunk);
+
+      if (chunk.type === 'response.created') {
+        // Verify response.created is the first event
+        expect(events.length).toBe(1);
+        expect(chunk.response.status).toBe('in_progress');
+        responseId = chunk.response.id;
+      } else if (chunk.type === 'response.completed') {
+        // Verify response.completed comes after response.created
+        expect(events.length).toBeGreaterThanOrEqual(2);
+        expect(chunk.response.status).toBe('completed');
+        expect(chunk.response.id).toBe(responseId);
+
+        // Verify content quality
+        const outputText = chunk.response.output_text.toLowerCase().trim();
+        expect(outputText.length).toBeGreaterThan(0);
+        expect(outputText).toContain(expected.toLowerCase());
+
+        // Verify usage is reported
+        expect(chunk.response.usage).toBeDefined();
+        expect(chunk.response.usage!.input_tokens).toBeGreaterThan(0);
+        expect(chunk.response.usage!.output_tokens).toBeGreaterThan(0);
+        expect(chunk.response.usage!.total_tokens).toBe(
+          chunk.response.usage!.input_tokens + chunk.response.usage!.output_tokens,
+        );
+      }
+    }
+
+    // Verify we got both events
+    expect(events.length).toBeGreaterThanOrEqual(2);
+    const firstEvent = events[0];
+    const lastEvent = events[events.length - 1];
+    expect(firstEvent.type).toBe('response.created');
+    expect(lastEvent.type).toBe('response.completed');
+
+    // Verify stored response matches streamed response
+    const retrievedResponse = await client.responses.retrieve(responseId);
+    expect(retrievedResponse.output_text).toBe(lastEvent.response.output_text);
+  });
+});

tests/integration/client-typescript/jest.integration.config.ts

@@ -0,0 +1,34 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+import type { JestConfigWithTsJest } from 'ts-jest';
+
+const config: JestConfigWithTsJest = {
+  preset: 'ts-jest/presets/default-esm',
+  testEnvironment: 'node',
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+        tsconfig: {
+          module: 'ES2022',
+          moduleResolution: 'bundler',
+        },
+      },
+    ],
+  },
+  testMatch: ['<rootDir>/__tests__/**/*.test.ts'],
+  setupFilesAfterEnv: ['<rootDir>/setup.ts'],
+  testTimeout: 60000, // 60 seconds (integration tests can be slow)
+  watchman: false, // Disable watchman to avoid permission issues
+};
+
+export default config;

tests/integration/client-typescript/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "llama-stack-typescript-integration-tests",
+  "version": "0.0.1",
+  "private": true,
+  "description": "TypeScript client integration tests for Llama Stack",
+  "type": "module",
+  "scripts": {
+    "test": "jest --config jest.integration.config.ts"
+  },
+  "dependencies": {
+    "llama-stack-client": "^0.3.2"
+  },
+  "devDependencies": {
+    "@swc/core": "^1.3.102",
+    "@swc/jest": "^0.2.29",
+    "@types/jest": "^29.4.0",
+    "@types/node": "^20.0.0",
+    "jest": "^29.4.0",
+    "ts-jest": "^29.1.0",
+    "typescript": "^5.0.0"
+  }
+}

tests/integration/client-typescript/setup.ts

@@ -0,0 +1,108 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+//
+// This source code is licensed under the terms described in the LICENSE file in
+// the root directory of this source tree.
+
+/**
+ * Global setup for integration tests.
+ * This file mimics pytest's fixture system by providing shared test configuration.
+ */
+
+import LlamaStackClient from 'llama-stack-client';
+
+// Read configuration from environment variables (set by scripts/integration-test.sh)
+export const TEST_CONFIG = {
+  baseURL: process.env['TEST_API_BASE_URL'],
+  textModel: process.env['LLAMA_STACK_TEST_MODEL'],
+  embeddingModel: process.env['LLAMA_STACK_TEST_EMBEDDING_MODEL'],
+} as const;
+
+// Validate required configuration
+beforeAll(() => {
+  if (!TEST_CONFIG.baseURL) {
+    throw new Error(
+      'TEST_API_BASE_URL is required for integration tests. ' +
+        'Run tests using: ./scripts/integration-test.sh',
+    );
+  }
+
+  console.log('\n=== Integration Test Configuration ===');
+  console.log(`Base URL: ${TEST_CONFIG.baseURL}`);
+  console.log(
+    `Text Model: ${TEST_CONFIG.textModel || 'NOT SET - tests requiring text model will be skipped'}`,
+  );
+  console.log(
+    `Embedding Model: ${
+      TEST_CONFIG.embeddingModel || 'NOT SET - tests requiring embedding model will be skipped'
+    }`,
+  );
+  console.log('=====================================\n');
+});
+
+/**
+ * Create a client instance for integration tests.
+ * Mimics pytest's `llama_stack_client` fixture.
+ *
+ * @param testId - Test ID to send in X-LlamaStack-Provider-Data header for replay mode.
+ *                 Format: "tests/integration/responses/test_basic_responses.py::test_name[params]"
+ */
+export function createTestClient(testId?: string): LlamaStackClient {
+  const headers: Record<string, string> = {};
+
+  // In server mode with replay, send test ID for recording isolation
+  if (process.env['LLAMA_STACK_TEST_STACK_CONFIG_TYPE'] === 'server' && testId) {
+    headers['X-LlamaStack-Provider-Data'] = JSON.stringify({
+      __test_id: testId,
+    });
+  }
+
+  return new LlamaStackClient({
+    baseURL: TEST_CONFIG.baseURL,
+    timeout: 60000, // 60 seconds
+    defaultHeaders: headers,
+  });
+}
+
+/**
+ * Skip test if required model is not configured.
+ * Mimics pytest's `skip_if_no_model` autouse fixture.
+ */
+export function skipIfNoModel(modelType: 'text' | 'embedding'): typeof test {
+  const model = modelType === 'text' ? TEST_CONFIG.textModel : TEST_CONFIG.embeddingModel;
+
+  if (!model) {
+    const message = `Skipping: ${modelType} model not configured (set LLAMA_STACK_TEST_${modelType.toUpperCase()}_MODEL)`;
+    return test.skip.bind(test) as typeof test;
+  }
+
+  return test;
+}
+
+/**
+ * Get the configured text model, throwing if not set.
+ * Use this in tests that absolutely require a text model.
+ */
+export function requireTextModel(): string {
+  if (!TEST_CONFIG.textModel) {
+    throw new Error(
+      'LLAMA_STACK_TEST_MODEL environment variable is required. ' +
+        'Run tests using: ./scripts/integration-test.sh',
+    );
+  }
+  return TEST_CONFIG.textModel;
+}
+
+/**
+ * Get the configured embedding model, throwing if not set.
+ * Use this in tests that absolutely require an embedding model.
+ */
+export function requireEmbeddingModel(): string {
+  if (!TEST_CONFIG.embeddingModel) {
+    throw new Error(
+      'LLAMA_STACK_TEST_EMBEDDING_MODEL environment variable is required. ' +
+        'Run tests using: ./scripts/integration-test.sh',
+    );
+  }
+  return TEST_CONFIG.embeddingModel;
+}

tests/integration/client-typescript/suites.json

@@ -0,0 +1,20 @@
+[
+  {
+    "suite": "responses",
+    "setup": "gpt",
+    "files": ["__tests__/responses.test.ts"]
+  },
+  {
+    "suite": "responses",
+    "files": ["__tests__/responses.test.ts"]
+  },
+  {
+    "suite": "inference",
+    "setup": "ollama",
+    "files": ["__tests__/inference.test.ts"]
+  },
+  {
+    "suite": "inference",
+    "files": ["__tests__/inference.test.ts"]
+  }
+]

tests/integration/client-typescript/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ES2022",
+    "lib": ["ES2022"],
+    "moduleResolution": "bundler",
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true,
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["node_modules"]
+}