feat(tests): add TypeScript client integration test support

Integration tests now support replaying TypeScript SDK tests alongside Python tests when running against server-mode stacks. This enables cross-language validation of API contracts and ensures the TypeScript client properly handles recorded responses.

The implementation adds a new `RUN_CLIENT_TS_TESTS` environment variable that triggers TypeScript test execution after successful Python runs. A mapping file (`suites.json`) defines which TypeScript test files correspond to each Python test suite/setup combination. The script automatically installs npm dependencies, forwards server configuration (base URL and model defaults from setup definitions), and executes matching TypeScript tests using Jest.

CI integration is enabled for server-based test jobs, and the feature can be exercised locally with commands like:

\`\`\`bash
RUN_CLIENT_TS_TESTS=1 scripts/integration-tests.sh --stack-config server:ci-tests --suite responses --setup gpt
\`\`\`

The TypeScript tests reuse existing replay fixtures through the forwarded \`TEST_API_BASE_URL\`, avoiding the need for duplicate response recordings.
This commit is contained in:
Ashwin Bharambe 2025-11-18 14:17:40 -08:00
parent 91f1b352b4
commit 78a676e231
12 changed files with 6082 additions and 0 deletions

View file

@ -93,11 +93,20 @@ jobs:
suite: ${{ matrix.config.suite }}
inference-mode: 'replay'
- name: Setup Node.js for TypeScript client tests
if: ${{ matrix.client == 'server' }}
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'npm'
cache-dependency-path: tests/integration/client-typescript/package-lock.json
- name: Run tests
if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }}
uses: ./.github/actions/run-and-record-tests
env:
OPENAI_API_KEY: dummy
RUN_CLIENT_TS_TESTS: ${{ matrix.client == 'server' && '1' || '0' }}
with:
stack-config: >-
${{ matrix.config.stack_config

1
.gitignore vendored
View file

@ -35,3 +35,4 @@ docs/static/imported-files/
docs/docs/api-deprecated/
docs/docs/api-experimental/
docs/docs/api/
tests/integration/client-typescript/node_modules/

View file

@ -20,6 +20,7 @@ TEST_PATTERN=""
INFERENCE_MODE="replay"
EXTRA_PARAMS=""
COLLECT_ONLY=false
RUN_CLIENT_TS_TESTS="${RUN_CLIENT_TS_TESTS:-0}"
# Function to display usage
usage() {
@ -120,6 +121,22 @@ if [[ -z "$TEST_SUITE" && -z "$TEST_SUBDIRS" ]]; then
exit 1
fi
RESOLVED_TEST_SETUP=$(python - "$TEST_SUITE" "$TEST_SETUP" <<'PY'
import sys
from tests.integration.suites import SUITE_DEFINITIONS
suite = sys.argv[1]
setup = sys.argv[2]
if not setup:
suite_def = SUITE_DEFINITIONS.get(suite)
if suite_def:
setup = suite_def.default_setup or ""
print(setup or "")
PY
)
echo "=== Llama Stack Integration Test Runner ==="
echo "Stack Config: $STACK_CONFIG"
echo "Setup: $TEST_SETUP"
@ -180,6 +197,38 @@ echo "Setting up environment variables:"
echo "$SETUP_ENV"
eval "$SETUP_ENV"
echo ""
if [[ -n "$RESOLVED_TEST_SETUP" ]]; then
SETUP_DEFAULTS=$(PYTHONPATH=$THIS_DIR/.. python - "$RESOLVED_TEST_SETUP" <<'PY'
import sys
from tests.integration.suites import SETUP_DEFINITIONS
setup_name = sys.argv[1]
if not setup_name:
sys.exit(0)
setup = SETUP_DEFINITIONS.get(setup_name)
if not setup:
sys.exit(0)
for key, value in setup.defaults.items():
print(f"{key}={value}")
PY
)
while IFS='=' read -r key value; do
case "$key" in
text_model)
export LLAMA_STACK_TEST_MODEL="$value"
;;
embedding_model)
export LLAMA_STACK_TEST_EMBEDDING_MODEL="$value"
;;
vision_model)
export LLAMA_STACK_TEST_VISION_MODEL="$value"
;;
esac
done <<< "$SETUP_DEFAULTS"
fi
ROOT_DIR="$THIS_DIR/.."
cd $ROOT_DIR
@ -212,6 +261,34 @@ find_available_port() {
return 1
}
run_client_ts_tests() {
local files=("$@")
if [[ ${#files[@]} -eq 0 ]]; then
echo "No TypeScript integration tests mapped for suite $TEST_SUITE (setup $RESOLVED_TEST_SETUP)"
return 0
fi
if ! command -v npm &>/dev/null; then
echo "npm could not be found; ensure Node.js is installed"
return 1
fi
pushd tests/integration/client-typescript >/dev/null
local install_cmd="npm install"
if [[ "${CI:-}" == "true" || "${CI:-}" == "1" ]]; then
install_cmd="npm ci"
fi
echo "Installing TypeScript client test dependencies using: $install_cmd"
$install_cmd
echo "Running TypeScript tests: ${files[*]}"
npx jest --config jest.integration.config.ts "${files[@]}"
popd >/dev/null
}
# Start Llama Stack Server if needed
if [[ "$STACK_CONFIG" == *"server:"* && "$COLLECT_ONLY" == false ]]; then
# Find an available port for the server
@ -221,6 +298,7 @@ if [[ "$STACK_CONFIG" == *"server:"* && "$COLLECT_ONLY" == false ]]; then
exit 1
fi
export LLAMA_STACK_PORT
export TEST_API_BASE_URL="http://localhost:$LLAMA_STACK_PORT"
echo "Will use port: $LLAMA_STACK_PORT"
stop_server() {
@ -298,6 +376,7 @@ if [[ "$STACK_CONFIG" == *"docker:"* && "$COLLECT_ONLY" == false ]]; then
exit 1
fi
export LLAMA_STACK_PORT
export TEST_API_BASE_URL="http://localhost:$LLAMA_STACK_PORT"
echo "Will use port: $LLAMA_STACK_PORT"
echo "=== Building Docker Image for distribution: $DISTRO ==="
@ -506,5 +585,45 @@ else
exit 1
fi
if [[ $exit_code -eq 0 && "$RUN_CLIENT_TS_TESTS" == "1" && "${LLAMA_STACK_TEST_STACK_CONFIG_TYPE:-}" == "server" ]]; then
CLIENT_TS_FILES=$(python - "$TEST_SUITE" "$RESOLVED_TEST_SETUP" <<'PY'
from pathlib import Path
import json
import sys
suite = sys.argv[1] or ""
setup = sys.argv[2] or ""
config_path = Path("tests/integration/client-typescript/suites.json")
if not config_path.exists():
sys.exit(0)
config = json.loads(config_path.read_text())
for entry in config:
if entry.get("suite") != suite:
continue
entry_setup = entry.get("setup") or ""
if entry_setup and entry_setup != setup:
continue
for file_name in entry.get("files", []):
print(file_name)
break
PY
)
if [[ -n "$CLIENT_TS_FILES" ]]; then
echo "Running TypeScript client tests for suite $TEST_SUITE (setup $RESOLVED_TEST_SETUP)"
CLIENT_TS_FILE_ARRAY=()
while IFS= read -r file; do
[[ -z "$file" ]] && continue
CLIENT_TS_FILE_ARRAY+=("$file")
done <<< "$CLIENT_TS_FILES"
run_client_ts_tests "${CLIENT_TS_FILE_ARRAY[@]}"
else
echo "No TypeScript client tests configured for suite $TEST_SUITE and setup $RESOLVED_TEST_SETUP"
fi
fi
echo ""
echo "=== Integration Tests Complete ==="

View file

@ -211,3 +211,13 @@ def test_asymmetric_embeddings(llama_stack_client, embedding_model_id):
assert query_response.embeddings is not None
```
## TypeScript Client Replays
Setting `RUN_CLIENT_TS_TESTS=1` when running `scripts/integration-tests.sh` against a `server:<config>` stack will replay the matching TypeScript SDK suites from `tests/integration/client-typescript/` immediately after the Python run. The mapping between suites/setups and `.test.ts` files lives in `tests/integration/client-typescript/suites.json`. This mode is enabled in CI for the `server` client jobs, and you can exercise it locally with commands such as:
```bash
RUN_CLIENT_TS_TESTS=1 scripts/integration-tests.sh --stack-config server:ci-tests --suite responses --setup gpt
```
The script installs the npm project on demand and forwards the server's `TEST_API_BASE_URL` + model defaults so the TypeScript tests can reuse the existing replay fixtures.

View file

@ -0,0 +1,104 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
// All rights reserved.
//
// This source code is licensed under the terms described in the LICENSE file in
// the root directory of this source tree.
/**
* Integration tests for Inference API (Chat Completions).
* Ported from: llama-stack/tests/integration/inference/test_openai_completion.py
*
* IMPORTANT: Test cases must match EXACTLY with Python tests to use recorded API responses.
*/
import { createTestClient, requireTextModel } from '../setup';
describe('Inference API - Chat Completions', () => {
// Test cases matching llama-stack/tests/integration/test_cases/inference/chat_completion.json
const chatCompletionTestCases = [
{
id: 'non_streaming_01',
question: 'Which planet do humans live on?',
expected: 'earth',
testId:
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_non_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:non_streaming_01]',
},
{
id: 'non_streaming_02',
question: 'Which planet has rings around it with a name starting with letter S?',
expected: 'saturn',
testId:
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_non_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:non_streaming_02]',
},
];
const streamingTestCases = [
{
id: 'streaming_01',
question: "What's the name of the Sun in latin?",
expected: 'sol',
testId:
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:streaming_01]',
},
{
id: 'streaming_02',
question: 'What is the name of the US captial?',
expected: 'washington',
testId:
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:streaming_02]',
},
];
test.each(chatCompletionTestCases)(
'chat completion non-streaming: $id',
async ({ question, expected, testId }) => {
const client = createTestClient(testId);
const textModel = requireTextModel();
const response = await client.chat.completions.create({
model: textModel,
messages: [
{
role: 'user',
content: question,
},
],
stream: false,
});
// Non-streaming responses have choices with message property
const choice = response.choices[0];
expect(choice).toBeDefined();
if (!choice || !('message' in choice)) {
throw new Error('Expected non-streaming response with message');
}
const content = choice.message.content;
expect(content).toBeDefined();
const messageContent = typeof content === 'string' ? content.toLowerCase().trim() : '';
expect(messageContent.length).toBeGreaterThan(0);
expect(messageContent).toContain(expected.toLowerCase());
},
);
test.each(streamingTestCases)('chat completion streaming: $id', async ({ question, expected, testId }) => {
const client = createTestClient(testId);
const textModel = requireTextModel();
const stream = await client.chat.completions.create({
model: textModel,
messages: [{ role: 'user', content: question }],
stream: true,
});
const streamedContent: string[] = [];
for await (const chunk of stream) {
if (chunk.choices && chunk.choices.length > 0 && chunk.choices[0]?.delta?.content) {
streamedContent.push(chunk.choices[0].delta.content);
}
}
expect(streamedContent.length).toBeGreaterThan(0);
const fullContent = streamedContent.join('').toLowerCase().trim();
expect(fullContent).toContain(expected.toLowerCase());
});
});

View file

@ -0,0 +1,132 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
// All rights reserved.
//
// This source code is licensed under the terms described in the LICENSE file in
// the root directory of this source tree.
/**
* Integration tests for Responses API.
* Ported from: llama-stack/tests/integration/responses/test_basic_responses.py
*
* IMPORTANT: Test cases and IDs must match EXACTLY with Python tests to use recorded API responses.
*/
import { createTestClient, requireTextModel } from '../setup';
describe('Responses API - Basic', () => {
// Test cases matching llama-stack/tests/integration/responses/fixtures/test_cases.py
const basicTestCases = [
{
id: 'earth',
input: 'Which planet do humans live on?',
expected: 'earth',
// Use client_with_models fixture to match non-streaming recordings
testId:
'tests/integration/responses/test_basic_responses.py::test_response_non_streaming_basic[client_with_models-txt=openai/gpt-4o-earth]',
},
{
id: 'saturn',
input: 'Which planet has rings around it with a name starting with letter S?',
expected: 'saturn',
testId:
'tests/integration/responses/test_basic_responses.py::test_response_non_streaming_basic[client_with_models-txt=openai/gpt-4o-saturn]',
},
];
test.each(basicTestCases)('non-streaming basic response: $id', async ({ input, expected, testId }) => {
// Create client with test_id for all requests
const client = createTestClient(testId);
const textModel = requireTextModel();
// Create a response
const response = await client.responses.create({
model: textModel,
input,
stream: false,
});
// Verify response has content
const outputText = response.output_text.toLowerCase().trim();
expect(outputText.length).toBeGreaterThan(0);
expect(outputText).toContain(expected.toLowerCase());
// Verify usage is reported
expect(response.usage).toBeDefined();
expect(response.usage!.input_tokens).toBeGreaterThan(0);
expect(response.usage!.output_tokens).toBeGreaterThan(0);
expect(response.usage!.total_tokens).toBe(response.usage!.input_tokens + response.usage!.output_tokens);
// Verify stored response matches
const retrievedResponse = await client.responses.retrieve(response.id);
expect(retrievedResponse.output_text).toBe(response.output_text);
// Test follow-up with previous_response_id
const nextResponse = await client.responses.create({
model: textModel,
input: 'Repeat your previous response in all caps.',
previous_response_id: response.id,
});
const nextOutputText = nextResponse.output_text.trim();
expect(nextOutputText).toContain(expected.toUpperCase());
});
test.each(basicTestCases)('streaming basic response: $id', async ({ input, expected, testId }) => {
// Modify test_id for streaming variant
const streamingTestId = testId.replace(
'test_response_non_streaming_basic',
'test_response_streaming_basic',
);
const client = createTestClient(streamingTestId);
const textModel = requireTextModel();
// Create a streaming response
const stream = await client.responses.create({
model: textModel,
input,
stream: true,
});
const events: any[] = [];
let responseId = '';
for await (const chunk of stream) {
events.push(chunk);
if (chunk.type === 'response.created') {
// Verify response.created is the first event
expect(events.length).toBe(1);
expect(chunk.response.status).toBe('in_progress');
responseId = chunk.response.id;
} else if (chunk.type === 'response.completed') {
// Verify response.completed comes after response.created
expect(events.length).toBeGreaterThanOrEqual(2);
expect(chunk.response.status).toBe('completed');
expect(chunk.response.id).toBe(responseId);
// Verify content quality
const outputText = chunk.response.output_text.toLowerCase().trim();
expect(outputText.length).toBeGreaterThan(0);
expect(outputText).toContain(expected.toLowerCase());
// Verify usage is reported
expect(chunk.response.usage).toBeDefined();
expect(chunk.response.usage!.input_tokens).toBeGreaterThan(0);
expect(chunk.response.usage!.output_tokens).toBeGreaterThan(0);
expect(chunk.response.usage!.total_tokens).toBe(
chunk.response.usage!.input_tokens + chunk.response.usage!.output_tokens,
);
}
}
// Verify we got both events
expect(events.length).toBeGreaterThanOrEqual(2);
const firstEvent = events[0];
const lastEvent = events[events.length - 1];
expect(firstEvent.type).toBe('response.created');
expect(lastEvent.type).toBe('response.completed');
// Verify stored response matches streamed response
const retrievedResponse = await client.responses.retrieve(responseId);
expect(retrievedResponse.output_text).toBe(lastEvent.response.output_text);
});
});

View file

@ -0,0 +1,34 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
// All rights reserved.
//
// This source code is licensed under the terms described in the LICENSE file in
// the root directory of this source tree.
import type { JestConfigWithTsJest } from 'ts-jest';
const config: JestConfigWithTsJest = {
preset: 'ts-jest/presets/default-esm',
testEnvironment: 'node',
extensionsToTreatAsEsm: ['.ts'],
moduleNameMapper: {
'^(\\.{1,2}/.*)\\.js$': '$1',
},
transform: {
'^.+\\.tsx?$': [
'ts-jest',
{
useESM: true,
tsconfig: {
module: 'ES2022',
moduleResolution: 'bundler',
},
},
],
},
testMatch: ['<rootDir>/__tests__/**/*.test.ts'],
setupFilesAfterEnv: ['<rootDir>/setup.ts'],
testTimeout: 60000, // 60 seconds (integration tests can be slow)
watchman: false, // Disable watchman to avoid permission issues
};
export default config;

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,22 @@
{
"name": "llama-stack-typescript-integration-tests",
"version": "0.0.1",
"private": true,
"description": "TypeScript client integration tests for Llama Stack",
"type": "module",
"scripts": {
"test": "jest --config jest.integration.config.ts"
},
"dependencies": {
"llama-stack-client": "^0.3.2"
},
"devDependencies": {
"@swc/core": "^1.3.102",
"@swc/jest": "^0.2.29",
"@types/jest": "^29.4.0",
"@types/node": "^20.0.0",
"jest": "^29.4.0",
"ts-jest": "^29.1.0",
"typescript": "^5.0.0"
}
}

View file

@ -0,0 +1,108 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
// All rights reserved.
//
// This source code is licensed under the terms described in the LICENSE file in
// the root directory of this source tree.
/**
* Global setup for integration tests.
* This file mimics pytest's fixture system by providing shared test configuration.
*/
import LlamaStackClient from 'llama-stack-client';
// Read configuration from environment variables (set by scripts/integration-test.sh)
export const TEST_CONFIG = {
baseURL: process.env['TEST_API_BASE_URL'],
textModel: process.env['LLAMA_STACK_TEST_MODEL'],
embeddingModel: process.env['LLAMA_STACK_TEST_EMBEDDING_MODEL'],
} as const;
// Validate required configuration
beforeAll(() => {
if (!TEST_CONFIG.baseURL) {
throw new Error(
'TEST_API_BASE_URL is required for integration tests. ' +
'Run tests using: ./scripts/integration-test.sh',
);
}
console.log('\n=== Integration Test Configuration ===');
console.log(`Base URL: ${TEST_CONFIG.baseURL}`);
console.log(
`Text Model: ${TEST_CONFIG.textModel || 'NOT SET - tests requiring text model will be skipped'}`,
);
console.log(
`Embedding Model: ${
TEST_CONFIG.embeddingModel || 'NOT SET - tests requiring embedding model will be skipped'
}`,
);
console.log('=====================================\n');
});
/**
* Create a client instance for integration tests.
* Mimics pytest's `llama_stack_client` fixture.
*
* @param testId - Test ID to send in X-LlamaStack-Provider-Data header for replay mode.
* Format: "tests/integration/responses/test_basic_responses.py::test_name[params]"
*/
export function createTestClient(testId?: string): LlamaStackClient {
const headers: Record<string, string> = {};
// In server mode with replay, send test ID for recording isolation
if (process.env['LLAMA_STACK_TEST_STACK_CONFIG_TYPE'] === 'server' && testId) {
headers['X-LlamaStack-Provider-Data'] = JSON.stringify({
__test_id: testId,
});
}
return new LlamaStackClient({
baseURL: TEST_CONFIG.baseURL,
timeout: 60000, // 60 seconds
defaultHeaders: headers,
});
}
/**
* Skip test if required model is not configured.
* Mimics pytest's `skip_if_no_model` autouse fixture.
*/
export function skipIfNoModel(modelType: 'text' | 'embedding'): typeof test {
const model = modelType === 'text' ? TEST_CONFIG.textModel : TEST_CONFIG.embeddingModel;
if (!model) {
const message = `Skipping: ${modelType} model not configured (set LLAMA_STACK_TEST_${modelType.toUpperCase()}_MODEL)`;
return test.skip.bind(test) as typeof test;
}
return test;
}
/**
* Get the configured text model, throwing if not set.
* Use this in tests that absolutely require a text model.
*/
export function requireTextModel(): string {
if (!TEST_CONFIG.textModel) {
throw new Error(
'LLAMA_STACK_TEST_MODEL environment variable is required. ' +
'Run tests using: ./scripts/integration-test.sh',
);
}
return TEST_CONFIG.textModel;
}
/**
* Get the configured embedding model, throwing if not set.
* Use this in tests that absolutely require an embedding model.
*/
export function requireEmbeddingModel(): string {
if (!TEST_CONFIG.embeddingModel) {
throw new Error(
'LLAMA_STACK_TEST_EMBEDDING_MODEL environment variable is required. ' +
'Run tests using: ./scripts/integration-test.sh',
);
}
return TEST_CONFIG.embeddingModel;
}

View file

@ -0,0 +1,20 @@
[
{
"suite": "responses",
"setup": "gpt",
"files": ["__tests__/responses.test.ts"]
},
{
"suite": "responses",
"files": ["__tests__/responses.test.ts"]
},
{
"suite": "inference",
"setup": "ollama",
"files": ["__tests__/inference.test.ts"]
},
{
"suite": "inference",
"files": ["__tests__/inference.test.ts"]
}
]

View file

@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ES2022",
"lib": ["ES2022"],
"moduleResolution": "bundler",
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"types": ["jest", "node"]
},
"include": ["**/*.ts"],
"exclude": ["node_modules"]
}