mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-03 18:00:36 +00:00
feat(tests): add TypeScript client integration test support
Integration tests now support replaying TypeScript SDK tests alongside Python tests when running against server-mode stacks. This enables cross-language validation of API contracts and ensures the TypeScript client properly handles recorded responses. The implementation adds a new `RUN_CLIENT_TS_TESTS` environment variable that triggers TypeScript test execution after successful Python runs. A mapping file (`suites.json`) defines which TypeScript test files correspond to each Python test suite/setup combination. The script automatically installs npm dependencies, forwards server configuration (base URL and model defaults from setup definitions), and executes matching TypeScript tests using Jest. CI integration is enabled for server-based test jobs, and the feature can be exercised locally with commands like: \`\`\`bash RUN_CLIENT_TS_TESTS=1 scripts/integration-tests.sh --stack-config server:ci-tests --suite responses --setup gpt \`\`\` The TypeScript tests reuse existing replay fixtures through the forwarded \`TEST_API_BASE_URL\`, avoiding the need for duplicate response recordings.
This commit is contained in:
parent
91f1b352b4
commit
78a676e231
12 changed files with 6082 additions and 0 deletions
9
.github/workflows/integration-tests.yml
vendored
9
.github/workflows/integration-tests.yml
vendored
|
|
@ -93,11 +93,20 @@ jobs:
|
|||
suite: ${{ matrix.config.suite }}
|
||||
inference-mode: 'replay'
|
||||
|
||||
- name: Setup Node.js for TypeScript client tests
|
||||
if: ${{ matrix.client == 'server' }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: tests/integration/client-typescript/package-lock.json
|
||||
|
||||
- name: Run tests
|
||||
if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }}
|
||||
uses: ./.github/actions/run-and-record-tests
|
||||
env:
|
||||
OPENAI_API_KEY: dummy
|
||||
RUN_CLIENT_TS_TESTS: ${{ matrix.client == 'server' && '1' || '0' }}
|
||||
with:
|
||||
stack-config: >-
|
||||
${{ matrix.config.stack_config
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -35,3 +35,4 @@ docs/static/imported-files/
|
|||
docs/docs/api-deprecated/
|
||||
docs/docs/api-experimental/
|
||||
docs/docs/api/
|
||||
tests/integration/client-typescript/node_modules/
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ TEST_PATTERN=""
|
|||
INFERENCE_MODE="replay"
|
||||
EXTRA_PARAMS=""
|
||||
COLLECT_ONLY=false
|
||||
RUN_CLIENT_TS_TESTS="${RUN_CLIENT_TS_TESTS:-0}"
|
||||
|
||||
# Function to display usage
|
||||
usage() {
|
||||
|
|
@ -120,6 +121,22 @@ if [[ -z "$TEST_SUITE" && -z "$TEST_SUBDIRS" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
RESOLVED_TEST_SETUP=$(python - "$TEST_SUITE" "$TEST_SETUP" <<'PY'
|
||||
import sys
|
||||
from tests.integration.suites import SUITE_DEFINITIONS
|
||||
|
||||
suite = sys.argv[1]
|
||||
setup = sys.argv[2]
|
||||
|
||||
if not setup:
|
||||
suite_def = SUITE_DEFINITIONS.get(suite)
|
||||
if suite_def:
|
||||
setup = suite_def.default_setup or ""
|
||||
|
||||
print(setup or "")
|
||||
PY
|
||||
)
|
||||
|
||||
echo "=== Llama Stack Integration Test Runner ==="
|
||||
echo "Stack Config: $STACK_CONFIG"
|
||||
echo "Setup: $TEST_SETUP"
|
||||
|
|
@ -180,6 +197,38 @@ echo "Setting up environment variables:"
|
|||
echo "$SETUP_ENV"
|
||||
eval "$SETUP_ENV"
|
||||
echo ""
|
||||
if [[ -n "$RESOLVED_TEST_SETUP" ]]; then
|
||||
SETUP_DEFAULTS=$(PYTHONPATH=$THIS_DIR/.. python - "$RESOLVED_TEST_SETUP" <<'PY'
|
||||
import sys
|
||||
from tests.integration.suites import SETUP_DEFINITIONS
|
||||
|
||||
setup_name = sys.argv[1]
|
||||
if not setup_name:
|
||||
sys.exit(0)
|
||||
|
||||
setup = SETUP_DEFINITIONS.get(setup_name)
|
||||
if not setup:
|
||||
sys.exit(0)
|
||||
|
||||
for key, value in setup.defaults.items():
|
||||
print(f"{key}={value}")
|
||||
PY
|
||||
)
|
||||
|
||||
while IFS='=' read -r key value; do
|
||||
case "$key" in
|
||||
text_model)
|
||||
export LLAMA_STACK_TEST_MODEL="$value"
|
||||
;;
|
||||
embedding_model)
|
||||
export LLAMA_STACK_TEST_EMBEDDING_MODEL="$value"
|
||||
;;
|
||||
vision_model)
|
||||
export LLAMA_STACK_TEST_VISION_MODEL="$value"
|
||||
;;
|
||||
esac
|
||||
done <<< "$SETUP_DEFAULTS"
|
||||
fi
|
||||
|
||||
ROOT_DIR="$THIS_DIR/.."
|
||||
cd $ROOT_DIR
|
||||
|
|
@ -212,6 +261,34 @@ find_available_port() {
|
|||
return 1
|
||||
}
|
||||
|
||||
run_client_ts_tests() {
|
||||
local files=("$@")
|
||||
|
||||
if [[ ${#files[@]} -eq 0 ]]; then
|
||||
echo "No TypeScript integration tests mapped for suite $TEST_SUITE (setup $RESOLVED_TEST_SETUP)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! command -v npm &>/dev/null; then
|
||||
echo "npm could not be found; ensure Node.js is installed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
pushd tests/integration/client-typescript >/dev/null
|
||||
|
||||
local install_cmd="npm install"
|
||||
if [[ "${CI:-}" == "true" || "${CI:-}" == "1" ]]; then
|
||||
install_cmd="npm ci"
|
||||
fi
|
||||
|
||||
echo "Installing TypeScript client test dependencies using: $install_cmd"
|
||||
$install_cmd
|
||||
echo "Running TypeScript tests: ${files[*]}"
|
||||
npx jest --config jest.integration.config.ts "${files[@]}"
|
||||
|
||||
popd >/dev/null
|
||||
}
|
||||
|
||||
# Start Llama Stack Server if needed
|
||||
if [[ "$STACK_CONFIG" == *"server:"* && "$COLLECT_ONLY" == false ]]; then
|
||||
# Find an available port for the server
|
||||
|
|
@ -221,6 +298,7 @@ if [[ "$STACK_CONFIG" == *"server:"* && "$COLLECT_ONLY" == false ]]; then
|
|||
exit 1
|
||||
fi
|
||||
export LLAMA_STACK_PORT
|
||||
export TEST_API_BASE_URL="http://localhost:$LLAMA_STACK_PORT"
|
||||
echo "Will use port: $LLAMA_STACK_PORT"
|
||||
|
||||
stop_server() {
|
||||
|
|
@ -298,6 +376,7 @@ if [[ "$STACK_CONFIG" == *"docker:"* && "$COLLECT_ONLY" == false ]]; then
|
|||
exit 1
|
||||
fi
|
||||
export LLAMA_STACK_PORT
|
||||
export TEST_API_BASE_URL="http://localhost:$LLAMA_STACK_PORT"
|
||||
echo "Will use port: $LLAMA_STACK_PORT"
|
||||
|
||||
echo "=== Building Docker Image for distribution: $DISTRO ==="
|
||||
|
|
@ -506,5 +585,45 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $exit_code -eq 0 && "$RUN_CLIENT_TS_TESTS" == "1" && "${LLAMA_STACK_TEST_STACK_CONFIG_TYPE:-}" == "server" ]]; then
|
||||
CLIENT_TS_FILES=$(python - "$TEST_SUITE" "$RESOLVED_TEST_SETUP" <<'PY'
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
suite = sys.argv[1] or ""
|
||||
setup = sys.argv[2] or ""
|
||||
|
||||
config_path = Path("tests/integration/client-typescript/suites.json")
|
||||
if not config_path.exists():
|
||||
sys.exit(0)
|
||||
|
||||
config = json.loads(config_path.read_text())
|
||||
|
||||
for entry in config:
|
||||
if entry.get("suite") != suite:
|
||||
continue
|
||||
entry_setup = entry.get("setup") or ""
|
||||
if entry_setup and entry_setup != setup:
|
||||
continue
|
||||
for file_name in entry.get("files", []):
|
||||
print(file_name)
|
||||
break
|
||||
PY
|
||||
)
|
||||
|
||||
if [[ -n "$CLIENT_TS_FILES" ]]; then
|
||||
echo "Running TypeScript client tests for suite $TEST_SUITE (setup $RESOLVED_TEST_SETUP)"
|
||||
CLIENT_TS_FILE_ARRAY=()
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
CLIENT_TS_FILE_ARRAY+=("$file")
|
||||
done <<< "$CLIENT_TS_FILES"
|
||||
run_client_ts_tests "${CLIENT_TS_FILE_ARRAY[@]}"
|
||||
else
|
||||
echo "No TypeScript client tests configured for suite $TEST_SUITE and setup $RESOLVED_TEST_SETUP"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Integration Tests Complete ==="
|
||||
|
|
|
|||
|
|
@ -211,3 +211,13 @@ def test_asymmetric_embeddings(llama_stack_client, embedding_model_id):
|
|||
|
||||
assert query_response.embeddings is not None
|
||||
```
|
||||
|
||||
## TypeScript Client Replays
|
||||
|
||||
Setting `RUN_CLIENT_TS_TESTS=1` when running `scripts/integration-tests.sh` against a `server:<config>` stack will replay the matching TypeScript SDK suites from `tests/integration/client-typescript/` immediately after the Python run. The mapping between suites/setups and `.test.ts` files lives in `tests/integration/client-typescript/suites.json`. This mode is enabled in CI for the `server` client jobs, and you can exercise it locally with commands such as:
|
||||
|
||||
```bash
|
||||
RUN_CLIENT_TS_TESTS=1 scripts/integration-tests.sh --stack-config server:ci-tests --suite responses --setup gpt
|
||||
```
|
||||
|
||||
The script installs the npm project on demand and forwards the server's `TEST_API_BASE_URL` + model defaults so the TypeScript tests can reuse the existing replay fixtures.
|
||||
|
|
|
|||
104
tests/integration/client-typescript/__tests__/inference.test.ts
Normal file
104
tests/integration/client-typescript/__tests__/inference.test.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// All rights reserved.
|
||||
//
|
||||
// This source code is licensed under the terms described in the LICENSE file in
|
||||
// the root directory of this source tree.
|
||||
|
||||
/**
|
||||
* Integration tests for Inference API (Chat Completions).
|
||||
* Ported from: llama-stack/tests/integration/inference/test_openai_completion.py
|
||||
*
|
||||
* IMPORTANT: Test cases must match EXACTLY with Python tests to use recorded API responses.
|
||||
*/
|
||||
|
||||
import { createTestClient, requireTextModel } from '../setup';
|
||||
|
||||
describe('Inference API - Chat Completions', () => {
|
||||
// Test cases matching llama-stack/tests/integration/test_cases/inference/chat_completion.json
|
||||
const chatCompletionTestCases = [
|
||||
{
|
||||
id: 'non_streaming_01',
|
||||
question: 'Which planet do humans live on?',
|
||||
expected: 'earth',
|
||||
testId:
|
||||
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_non_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:non_streaming_01]',
|
||||
},
|
||||
{
|
||||
id: 'non_streaming_02',
|
||||
question: 'Which planet has rings around it with a name starting with letter S?',
|
||||
expected: 'saturn',
|
||||
testId:
|
||||
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_non_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:non_streaming_02]',
|
||||
},
|
||||
];
|
||||
|
||||
const streamingTestCases = [
|
||||
{
|
||||
id: 'streaming_01',
|
||||
question: "What's the name of the Sun in latin?",
|
||||
expected: 'sol',
|
||||
testId:
|
||||
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:streaming_01]',
|
||||
},
|
||||
{
|
||||
id: 'streaming_02',
|
||||
question: 'What is the name of the US captial?',
|
||||
expected: 'washington',
|
||||
testId:
|
||||
'tests/integration/inference/test_openai_completion.py::test_openai_chat_completion_streaming[client_with_models-txt=ollama/llama3.2:3b-instruct-fp16-inference:chat_completion:streaming_02]',
|
||||
},
|
||||
];
|
||||
|
||||
test.each(chatCompletionTestCases)(
|
||||
'chat completion non-streaming: $id',
|
||||
async ({ question, expected, testId }) => {
|
||||
const client = createTestClient(testId);
|
||||
const textModel = requireTextModel();
|
||||
|
||||
const response = await client.chat.completions.create({
|
||||
model: textModel,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: question,
|
||||
},
|
||||
],
|
||||
stream: false,
|
||||
});
|
||||
|
||||
// Non-streaming responses have choices with message property
|
||||
const choice = response.choices[0];
|
||||
expect(choice).toBeDefined();
|
||||
if (!choice || !('message' in choice)) {
|
||||
throw new Error('Expected non-streaming response with message');
|
||||
}
|
||||
const content = choice.message.content;
|
||||
expect(content).toBeDefined();
|
||||
const messageContent = typeof content === 'string' ? content.toLowerCase().trim() : '';
|
||||
expect(messageContent.length).toBeGreaterThan(0);
|
||||
expect(messageContent).toContain(expected.toLowerCase());
|
||||
},
|
||||
);
|
||||
|
||||
test.each(streamingTestCases)('chat completion streaming: $id', async ({ question, expected, testId }) => {
|
||||
const client = createTestClient(testId);
|
||||
const textModel = requireTextModel();
|
||||
|
||||
const stream = await client.chat.completions.create({
|
||||
model: textModel,
|
||||
messages: [{ role: 'user', content: question }],
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const streamedContent: string[] = [];
|
||||
for await (const chunk of stream) {
|
||||
if (chunk.choices && chunk.choices.length > 0 && chunk.choices[0]?.delta?.content) {
|
||||
streamedContent.push(chunk.choices[0].delta.content);
|
||||
}
|
||||
}
|
||||
|
||||
expect(streamedContent.length).toBeGreaterThan(0);
|
||||
const fullContent = streamedContent.join('').toLowerCase().trim();
|
||||
expect(fullContent).toContain(expected.toLowerCase());
|
||||
});
|
||||
});
|
||||
132
tests/integration/client-typescript/__tests__/responses.test.ts
Normal file
132
tests/integration/client-typescript/__tests__/responses.test.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// All rights reserved.
|
||||
//
|
||||
// This source code is licensed under the terms described in the LICENSE file in
|
||||
// the root directory of this source tree.
|
||||
|
||||
/**
|
||||
* Integration tests for Responses API.
|
||||
* Ported from: llama-stack/tests/integration/responses/test_basic_responses.py
|
||||
*
|
||||
* IMPORTANT: Test cases and IDs must match EXACTLY with Python tests to use recorded API responses.
|
||||
*/
|
||||
|
||||
import { createTestClient, requireTextModel } from '../setup';
|
||||
|
||||
describe('Responses API - Basic', () => {
|
||||
// Test cases matching llama-stack/tests/integration/responses/fixtures/test_cases.py
|
||||
const basicTestCases = [
|
||||
{
|
||||
id: 'earth',
|
||||
input: 'Which planet do humans live on?',
|
||||
expected: 'earth',
|
||||
// Use client_with_models fixture to match non-streaming recordings
|
||||
testId:
|
||||
'tests/integration/responses/test_basic_responses.py::test_response_non_streaming_basic[client_with_models-txt=openai/gpt-4o-earth]',
|
||||
},
|
||||
{
|
||||
id: 'saturn',
|
||||
input: 'Which planet has rings around it with a name starting with letter S?',
|
||||
expected: 'saturn',
|
||||
testId:
|
||||
'tests/integration/responses/test_basic_responses.py::test_response_non_streaming_basic[client_with_models-txt=openai/gpt-4o-saturn]',
|
||||
},
|
||||
];
|
||||
|
||||
test.each(basicTestCases)('non-streaming basic response: $id', async ({ input, expected, testId }) => {
|
||||
// Create client with test_id for all requests
|
||||
const client = createTestClient(testId);
|
||||
const textModel = requireTextModel();
|
||||
|
||||
// Create a response
|
||||
const response = await client.responses.create({
|
||||
model: textModel,
|
||||
input,
|
||||
stream: false,
|
||||
});
|
||||
|
||||
// Verify response has content
|
||||
const outputText = response.output_text.toLowerCase().trim();
|
||||
expect(outputText.length).toBeGreaterThan(0);
|
||||
expect(outputText).toContain(expected.toLowerCase());
|
||||
|
||||
// Verify usage is reported
|
||||
expect(response.usage).toBeDefined();
|
||||
expect(response.usage!.input_tokens).toBeGreaterThan(0);
|
||||
expect(response.usage!.output_tokens).toBeGreaterThan(0);
|
||||
expect(response.usage!.total_tokens).toBe(response.usage!.input_tokens + response.usage!.output_tokens);
|
||||
|
||||
// Verify stored response matches
|
||||
const retrievedResponse = await client.responses.retrieve(response.id);
|
||||
expect(retrievedResponse.output_text).toBe(response.output_text);
|
||||
|
||||
// Test follow-up with previous_response_id
|
||||
const nextResponse = await client.responses.create({
|
||||
model: textModel,
|
||||
input: 'Repeat your previous response in all caps.',
|
||||
previous_response_id: response.id,
|
||||
});
|
||||
const nextOutputText = nextResponse.output_text.trim();
|
||||
expect(nextOutputText).toContain(expected.toUpperCase());
|
||||
});
|
||||
|
||||
test.each(basicTestCases)('streaming basic response: $id', async ({ input, expected, testId }) => {
|
||||
// Modify test_id for streaming variant
|
||||
const streamingTestId = testId.replace(
|
||||
'test_response_non_streaming_basic',
|
||||
'test_response_streaming_basic',
|
||||
);
|
||||
const client = createTestClient(streamingTestId);
|
||||
const textModel = requireTextModel();
|
||||
|
||||
// Create a streaming response
|
||||
const stream = await client.responses.create({
|
||||
model: textModel,
|
||||
input,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
const events: any[] = [];
|
||||
let responseId = '';
|
||||
|
||||
for await (const chunk of stream) {
|
||||
events.push(chunk);
|
||||
|
||||
if (chunk.type === 'response.created') {
|
||||
// Verify response.created is the first event
|
||||
expect(events.length).toBe(1);
|
||||
expect(chunk.response.status).toBe('in_progress');
|
||||
responseId = chunk.response.id;
|
||||
} else if (chunk.type === 'response.completed') {
|
||||
// Verify response.completed comes after response.created
|
||||
expect(events.length).toBeGreaterThanOrEqual(2);
|
||||
expect(chunk.response.status).toBe('completed');
|
||||
expect(chunk.response.id).toBe(responseId);
|
||||
|
||||
// Verify content quality
|
||||
const outputText = chunk.response.output_text.toLowerCase().trim();
|
||||
expect(outputText.length).toBeGreaterThan(0);
|
||||
expect(outputText).toContain(expected.toLowerCase());
|
||||
|
||||
// Verify usage is reported
|
||||
expect(chunk.response.usage).toBeDefined();
|
||||
expect(chunk.response.usage!.input_tokens).toBeGreaterThan(0);
|
||||
expect(chunk.response.usage!.output_tokens).toBeGreaterThan(0);
|
||||
expect(chunk.response.usage!.total_tokens).toBe(
|
||||
chunk.response.usage!.input_tokens + chunk.response.usage!.output_tokens,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify we got both events
|
||||
expect(events.length).toBeGreaterThanOrEqual(2);
|
||||
const firstEvent = events[0];
|
||||
const lastEvent = events[events.length - 1];
|
||||
expect(firstEvent.type).toBe('response.created');
|
||||
expect(lastEvent.type).toBe('response.completed');
|
||||
|
||||
// Verify stored response matches streamed response
|
||||
const retrievedResponse = await client.responses.retrieve(responseId);
|
||||
expect(retrievedResponse.output_text).toBe(lastEvent.response.output_text);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// All rights reserved.
|
||||
//
|
||||
// This source code is licensed under the terms described in the LICENSE file in
|
||||
// the root directory of this source tree.
|
||||
|
||||
import type { JestConfigWithTsJest } from 'ts-jest';
|
||||
|
||||
const config: JestConfigWithTsJest = {
|
||||
preset: 'ts-jest/presets/default-esm',
|
||||
testEnvironment: 'node',
|
||||
extensionsToTreatAsEsm: ['.ts'],
|
||||
moduleNameMapper: {
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
transform: {
|
||||
'^.+\\.tsx?$': [
|
||||
'ts-jest',
|
||||
{
|
||||
useESM: true,
|
||||
tsconfig: {
|
||||
module: 'ES2022',
|
||||
moduleResolution: 'bundler',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
testMatch: ['<rootDir>/__tests__/**/*.test.ts'],
|
||||
setupFilesAfterEnv: ['<rootDir>/setup.ts'],
|
||||
testTimeout: 60000, // 60 seconds (integration tests can be slow)
|
||||
watchman: false, // Disable watchman to avoid permission issues
|
||||
};
|
||||
|
||||
export default config;
|
||||
5507
tests/integration/client-typescript/package-lock.json
generated
Normal file
5507
tests/integration/client-typescript/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
22
tests/integration/client-typescript/package.json
Normal file
22
tests/integration/client-typescript/package.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"name": "llama-stack-typescript-integration-tests",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"description": "TypeScript client integration tests for Llama Stack",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "jest --config jest.integration.config.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"llama-stack-client": "^0.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@swc/core": "^1.3.102",
|
||||
"@swc/jest": "^0.2.29",
|
||||
"@types/jest": "^29.4.0",
|
||||
"@types/node": "^20.0.0",
|
||||
"jest": "^29.4.0",
|
||||
"ts-jest": "^29.1.0",
|
||||
"typescript": "^5.0.0"
|
||||
}
|
||||
}
|
||||
108
tests/integration/client-typescript/setup.ts
Normal file
108
tests/integration/client-typescript/setup.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
// All rights reserved.
|
||||
//
|
||||
// This source code is licensed under the terms described in the LICENSE file in
|
||||
// the root directory of this source tree.
|
||||
|
||||
/**
|
||||
* Global setup for integration tests.
|
||||
* This file mimics pytest's fixture system by providing shared test configuration.
|
||||
*/
|
||||
|
||||
import LlamaStackClient from 'llama-stack-client';
|
||||
|
||||
// Read configuration from environment variables (set by scripts/integration-test.sh)
|
||||
export const TEST_CONFIG = {
|
||||
baseURL: process.env['TEST_API_BASE_URL'],
|
||||
textModel: process.env['LLAMA_STACK_TEST_MODEL'],
|
||||
embeddingModel: process.env['LLAMA_STACK_TEST_EMBEDDING_MODEL'],
|
||||
} as const;
|
||||
|
||||
// Validate required configuration
|
||||
beforeAll(() => {
|
||||
if (!TEST_CONFIG.baseURL) {
|
||||
throw new Error(
|
||||
'TEST_API_BASE_URL is required for integration tests. ' +
|
||||
'Run tests using: ./scripts/integration-test.sh',
|
||||
);
|
||||
}
|
||||
|
||||
console.log('\n=== Integration Test Configuration ===');
|
||||
console.log(`Base URL: ${TEST_CONFIG.baseURL}`);
|
||||
console.log(
|
||||
`Text Model: ${TEST_CONFIG.textModel || 'NOT SET - tests requiring text model will be skipped'}`,
|
||||
);
|
||||
console.log(
|
||||
`Embedding Model: ${
|
||||
TEST_CONFIG.embeddingModel || 'NOT SET - tests requiring embedding model will be skipped'
|
||||
}`,
|
||||
);
|
||||
console.log('=====================================\n');
|
||||
});
|
||||
|
||||
/**
|
||||
* Create a client instance for integration tests.
|
||||
* Mimics pytest's `llama_stack_client` fixture.
|
||||
*
|
||||
* @param testId - Test ID to send in X-LlamaStack-Provider-Data header for replay mode.
|
||||
* Format: "tests/integration/responses/test_basic_responses.py::test_name[params]"
|
||||
*/
|
||||
export function createTestClient(testId?: string): LlamaStackClient {
|
||||
const headers: Record<string, string> = {};
|
||||
|
||||
// In server mode with replay, send test ID for recording isolation
|
||||
if (process.env['LLAMA_STACK_TEST_STACK_CONFIG_TYPE'] === 'server' && testId) {
|
||||
headers['X-LlamaStack-Provider-Data'] = JSON.stringify({
|
||||
__test_id: testId,
|
||||
});
|
||||
}
|
||||
|
||||
return new LlamaStackClient({
|
||||
baseURL: TEST_CONFIG.baseURL,
|
||||
timeout: 60000, // 60 seconds
|
||||
defaultHeaders: headers,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Skip test if required model is not configured.
|
||||
* Mimics pytest's `skip_if_no_model` autouse fixture.
|
||||
*/
|
||||
export function skipIfNoModel(modelType: 'text' | 'embedding'): typeof test {
|
||||
const model = modelType === 'text' ? TEST_CONFIG.textModel : TEST_CONFIG.embeddingModel;
|
||||
|
||||
if (!model) {
|
||||
const message = `Skipping: ${modelType} model not configured (set LLAMA_STACK_TEST_${modelType.toUpperCase()}_MODEL)`;
|
||||
return test.skip.bind(test) as typeof test;
|
||||
}
|
||||
|
||||
return test;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configured text model, throwing if not set.
|
||||
* Use this in tests that absolutely require a text model.
|
||||
*/
|
||||
export function requireTextModel(): string {
|
||||
if (!TEST_CONFIG.textModel) {
|
||||
throw new Error(
|
||||
'LLAMA_STACK_TEST_MODEL environment variable is required. ' +
|
||||
'Run tests using: ./scripts/integration-test.sh',
|
||||
);
|
||||
}
|
||||
return TEST_CONFIG.textModel;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the configured embedding model, throwing if not set.
|
||||
* Use this in tests that absolutely require an embedding model.
|
||||
*/
|
||||
export function requireEmbeddingModel(): string {
|
||||
if (!TEST_CONFIG.embeddingModel) {
|
||||
throw new Error(
|
||||
'LLAMA_STACK_TEST_EMBEDDING_MODEL environment variable is required. ' +
|
||||
'Run tests using: ./scripts/integration-test.sh',
|
||||
);
|
||||
}
|
||||
return TEST_CONFIG.embeddingModel;
|
||||
}
|
||||
20
tests/integration/client-typescript/suites.json
Normal file
20
tests/integration/client-typescript/suites.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
[
|
||||
{
|
||||
"suite": "responses",
|
||||
"setup": "gpt",
|
||||
"files": ["__tests__/responses.test.ts"]
|
||||
},
|
||||
{
|
||||
"suite": "responses",
|
||||
"files": ["__tests__/responses.test.ts"]
|
||||
},
|
||||
{
|
||||
"suite": "inference",
|
||||
"setup": "ollama",
|
||||
"files": ["__tests__/inference.test.ts"]
|
||||
},
|
||||
{
|
||||
"suite": "inference",
|
||||
"files": ["__tests__/inference.test.ts"]
|
||||
}
|
||||
]
|
||||
16
tests/integration/client-typescript/tsconfig.json
Normal file
16
tests/integration/client-typescript/tsconfig.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"lib": ["ES2022"],
|
||||
"moduleResolution": "bundler",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"types": ["jest", "node"]
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue