diff --git a/.circleci/config.yml b/.circleci/config.yml index 2aa6a1863..a23192b49 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -678,6 +678,100 @@ jobs: -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" echo "triggering stable release server for version ${VERSION} and commit ${CIRCLE_SHA1}" curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}" + + e2e_ui_testing: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + npm install -D @playwright/test + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + pip install "openai==1.45.0" + python -m pip install --upgrade pip + pip install "pydantic==2.7.1" + pip install "pytest==7.3.1" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install mypy + pip install pyarrow + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.24.1" + pip install "anyio==3.7.1" + pip install "asyncio==3.4.3" + - run: + name: Install Playwright Browsers + command: | + npx playwright install + - run: + name: Build Docker image + command: docker build -t my-app:latest -f Dockerfile.database . + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e UI_USERNAME="admin" \ + -e UI_PASSWORD="gm" \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run Playwright Tests + command: | + npx playwright test --reporter=html --output=test-results + no_output_timeout: 120m + - store_test_results: + path: test-results + workflows: version: 2 build_and_test: @@ -706,6 +800,12 @@ workflows: only: - main - /litellm_.*/ + - e2e_ui_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - build_and_test: filters: branches: @@ -751,6 +851,7 @@ workflows: - litellm_router_testing - litellm_assistants_api_testing - ui_endpoint_testing + - e2e_ui_testing - installing_litellm_on_python - proxy_logging_guardrails_model_info_tests - proxy_pass_through_endpoint_tests diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts new file mode 100644 index 000000000..730e5eec3 --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts @@ -0,0 +1,48 @@ +/* + +Login to Admin UI +Basic UI Test + +Click on all the tabs ensure nothing is broken +*/ + +import { test, expect } from '@playwright/test'; + +test('admin login test', async ({ page }) => { + // Go to the specified URL + await page.goto('http://localhost:4000/ui'); + + // Enter "admin" in the username input field + await page.fill('input[name="username"]', 'admin'); + + // Enter "gm" in the password input field + await page.fill('input[name="password"]', 'gm'); + + // Optionally, you can add an assertion to verify the login button is enabled + const loginButton = page.locator('input[type="submit"]'); + await expect(loginButton).toBeEnabled(); + + // Optionally, you can click the login button to submit the form + await loginButton.click(); + const tabs = [ + 'Virtual Keys', + 'Test Key', + 'Models', + 'Usage', + 'Teams', + 'Internal User', + 'Logging & Alerts', + 'Caching', + 'Budgets', + 'Router Settings', + 'Pass-through', + 'Admin Settings', + 'API Reference', + 'Model Hub' + ]; + + for (const tab of tabs) { + const tabElement = page.locator('p.text-tremor-default.text-tremor-content.dark\\:text-dark-tremor-content', { hasText: tab }); + await tabElement.click(); + } +}); diff --git a/tests/proxy_admin_ui_tests/package-lock.json b/tests/proxy_admin_ui_tests/package-lock.json new file mode 100644 index 000000000..3152ee9bf --- /dev/null +++ b/tests/proxy_admin_ui_tests/package-lock.json @@ -0,0 +1,91 @@ +{ + "name": "proxy_admin_ui_tests", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "proxy_admin_ui_tests", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.47.2", + "@types/node": "^22.5.5" + } + }, + "node_modules/@playwright/test": { + "version": "1.47.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.47.2.tgz", + "integrity": "sha512-jTXRsoSPONAs8Za9QEQdyjFn+0ZQFjCiIztAIF6bi1HqhBzG9Ma7g1WotyiGqFSBRZjIEqMdT8RUlbk1QVhzCQ==", + "dev": true, + "dependencies": { + "playwright": "1.47.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "22.5.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz", + "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==", + "dev": true, + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.47.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.47.2.tgz", + "integrity": "sha512-nx1cLMmQWqmA3UsnjaaokyoUpdVaaDhJhMoxX2qj3McpjnsqFHs516QAKYhqHAgOP+oCFTEOCOAaD1RgD/RQfA==", + "dev": true, + "dependencies": { + "playwright-core": "1.47.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.47.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.47.2.tgz", + "integrity": "sha512-3JvMfF+9LJfe16l7AbSmU555PaTl2tPyQsVInqm3id16pdDfvZ8TTZ/pyzmkbDrZTQefyzU7AIHlZqQnxpqHVQ==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true + } + } +} diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json new file mode 100644 index 000000000..20dfed7a8 --- /dev/null +++ b/tests/proxy_admin_ui_tests/package.json @@ -0,0 +1,14 @@ +{ + "name": "proxy_admin_ui_tests", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.47.2", + "@types/node": "^22.5.5" + } +} diff --git a/tests/proxy_admin_ui_tests/playwright.config.ts b/tests/proxy_admin_ui_tests/playwright.config.ts new file mode 100644 index 000000000..c77897a02 --- /dev/null +++ b/tests/proxy_admin_ui_tests/playwright.config.ts @@ -0,0 +1,79 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// import dotenv from 'dotenv'; +// import path from 'path'; +// dotenv.config({ path: path.resolve(__dirname, '.env') }); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './e2e_ui_tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +});