From 912080828a3cda43ad501874abe9cd6e1adf2ee2 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Tue, 18 Mar 2025 16:12:43 -0400 Subject: [PATCH 01/14] Added login e2e test for admin ui, using playwright --- .../tests/data/models-and-providers.json | 12 + .../tests/e2e/example.spec.ts | 18 + ui/litellm-dashboard/tests/e2e/login.spec.ts | 38 ++ .../tests/fixtures/fixtures.ts | 18 + ui/litellm-dashboard/tests/package-lock.json | 105 +++++ ui/litellm-dashboard/tests/package.json | 17 + .../tests/page-object-models/login.page.ts | 27 ++ .../page-object-models/virtual-keys.page.ts | 20 + .../tests/playwright.config.ts | 76 +++ .../tests-examples/demo-todo-app.spec.ts | 437 ++++++++++++++++++ .../tests/utils/globalSetup.ts | 8 + ui/litellm-dashboard/tests/utils/utils.ts | 14 + 12 files changed, 790 insertions(+) create mode 100644 ui/litellm-dashboard/tests/data/models-and-providers.json create mode 100644 ui/litellm-dashboard/tests/e2e/example.spec.ts create mode 100644 ui/litellm-dashboard/tests/e2e/login.spec.ts create mode 100644 ui/litellm-dashboard/tests/fixtures/fixtures.ts create mode 100644 ui/litellm-dashboard/tests/package-lock.json create mode 100644 ui/litellm-dashboard/tests/package.json create mode 100644 ui/litellm-dashboard/tests/page-object-models/login.page.ts create mode 100644 ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts create mode 100644 ui/litellm-dashboard/tests/playwright.config.ts create mode 100644 ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts create mode 100644 ui/litellm-dashboard/tests/utils/globalSetup.ts create mode 100644 ui/litellm-dashboard/tests/utils/utils.ts diff --git a/ui/litellm-dashboard/tests/data/models-and-providers.json b/ui/litellm-dashboard/tests/data/models-and-providers.json new file mode 100644 index 0000000000..1deb1d58cc --- /dev/null +++ b/ui/litellm-dashboard/tests/data/models-and-providers.json @@ -0,0 +1,12 @@ +{ + "openai": [ + "omni-moderation-latest", + "omni-moderation-latest-intents", + "omni-moderation-2024-09-26" + ], + "azure": [ + "azure/gpt-4o-mini-realtime-preview-2024-12-17", + "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17", + "azure/us/gpt-4o-mini-realtime-preview-2024-12-17" + ] +} diff --git a/ui/litellm-dashboard/tests/e2e/example.spec.ts b/ui/litellm-dashboard/tests/e2e/example.spec.ts new file mode 100644 index 0000000000..54a906a4e8 --- /dev/null +++ b/ui/litellm-dashboard/tests/e2e/example.spec.ts @@ -0,0 +1,18 @@ +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('https://playwright.dev/'); + + // Expect a title "to contain" a substring. + await expect(page).toHaveTitle(/Playwright/); +}); + +test('get started link', async ({ page }) => { + await page.goto('https://playwright.dev/'); + + // Click the get started link. + await page.getByRole('link', { name: 'Get started' }).click(); + + // Expects page to have a heading with the name of Installation. + await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible(); +}); diff --git a/ui/litellm-dashboard/tests/e2e/login.spec.ts b/ui/litellm-dashboard/tests/e2e/login.spec.ts new file mode 100644 index 0000000000..39057fa0bf --- /dev/null +++ b/ui/litellm-dashboard/tests/e2e/login.spec.ts @@ -0,0 +1,38 @@ +import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +// import { config } from "dotenv"; +// import path from "path"; +// config({ path: "./../../../../.env.example" }); + +test("Login", async ({ loginPage, virtualKeysPage, page }) => { + // Check if there are credentials in local environment file and assign credentials as appropriate. + // Login and verify user is directed to the dashboard. + // Do playwright tests run in order. + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + await loginPage.goto(); + await page.screenshot({ path: "./test-results/go-to-login-page.png" }); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + await page.screenshot({ path: "./test-results/dashboard.png" }); + + await virtualKeysPage.logout(); + await expect( + page.getByRole("heading", { name: "LiteLLM Login" }) + ).toBeVisible(); + await page.screenshot({ path: "./test-results/logout.png" }); +}); + +/* to dos +1. Add ui link to local env file +2. add login details to env file +3. configure screenshots and add path to screenshots +*/ diff --git a/ui/litellm-dashboard/tests/fixtures/fixtures.ts b/ui/litellm-dashboard/tests/fixtures/fixtures.ts new file mode 100644 index 0000000000..557bb2386b --- /dev/null +++ b/ui/litellm-dashboard/tests/fixtures/fixtures.ts @@ -0,0 +1,18 @@ +import { VirtualKeysPage } from "./../page-object-models/virtual-keys.page"; +import { test as base } from "@playwright/test"; +import { LoginPage } from "../page-object-models/login.page"; + +type Fixtures = { + loginPage: LoginPage; + virtualKeysPage: VirtualKeysPage; +}; + +export const test = base.extend({ + loginPage: async ({ page }, use) => { + await use(new LoginPage(page)); + }, + virtualKeysPage: async ({ page }, use) => { + await use(new VirtualKeysPage(page)); + }, +}); +export { expect } from "@playwright/test"; diff --git a/ui/litellm-dashboard/tests/package-lock.json b/ui/litellm-dashboard/tests/package-lock.json new file mode 100644 index 0000000000..19ade17417 --- /dev/null +++ b/ui/litellm-dashboard/tests/package-lock.json @@ -0,0 +1,105 @@ +{ + "name": "tests", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tests", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "dotenv": "^16.4.7" + }, + "devDependencies": { + "@playwright/test": "^1.51.0", + "@types/node": "^22.13.10" + } + }, + "node_modules/@playwright/test": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.0.tgz", + "integrity": "sha512-dJ0dMbZeHhI+wb77+ljx/FeC8VBP6j/rj9OAojO08JI80wTZy6vRk9KvHKiDCUh4iMpEiseMgqRBIeW+eKX6RA==", + "dev": true, + "dependencies": { + "playwright": "1.51.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "22.13.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz", + "integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==", + "dev": true, + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.0.tgz", + "integrity": "sha512-442pTfGM0xxfCYxuBa/Pu6B2OqxqqaYq39JS8QDMGThUvIOCd6s0ANDog3uwA0cHavVlnTQzGCN7Id2YekDSXA==", + "dev": true, + "dependencies": { + "playwright-core": "1.51.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.0.tgz", + "integrity": "sha512-x47yPE3Zwhlil7wlNU/iktF7t2r/URR3VLbH6EknJd/04Qc/PSJ0EY3CMXipmglLG+zyRxW6HNo2EGbKLHPWMg==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true + } + } +} diff --git a/ui/litellm-dashboard/tests/package.json b/ui/litellm-dashboard/tests/package.json new file mode 100644 index 0000000000..650bf45548 --- /dev/null +++ b/ui/litellm-dashboard/tests/package.json @@ -0,0 +1,17 @@ +{ + "name": "tests", + "version": "1.0.0", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "devDependencies": { + "@playwright/test": "^1.51.0", + "@types/node": "^22.13.10" + }, + "dependencies": { + "dotenv": "^16.4.7" + } +} diff --git a/ui/litellm-dashboard/tests/page-object-models/login.page.ts b/ui/litellm-dashboard/tests/page-object-models/login.page.ts new file mode 100644 index 0000000000..f0890c05f3 --- /dev/null +++ b/ui/litellm-dashboard/tests/page-object-models/login.page.ts @@ -0,0 +1,27 @@ +import type { Page, Locator } from "@playwright/test"; + +export class LoginPage { + //Locators as fields + private readonly usernameInput: Locator; + private readonly passwordInput: Locator; + private readonly loginSubmit: Locator; + + //Initialize locators in constructor + constructor(public readonly page: Page) { + this.usernameInput = this.page.getByRole("textbox", { name: "Username:" }); + this.passwordInput = this.page.getByRole("textbox", { name: "Password:" }); + this.loginSubmit = this.page.getByRole("button", { name: "Submit" }); + } + + async goto() { + await this.page.goto("/ui"); + } + + async login(username: string, password: string) { + await this.usernameInput.click(); + await this.usernameInput.fill(username); + await this.passwordInput.click(); + await this.passwordInput.fill(password); + await this.loginSubmit.click(); + } +} diff --git a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts new file mode 100644 index 0000000000..7b1fb72040 --- /dev/null +++ b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts @@ -0,0 +1,20 @@ +import type { Page, Locator } from "@playwright/test"; + +export class VirtualKeysPage { + private readonly userButton: Locator; + private readonly logoutButton: Locator; + + constructor(public readonly page: Page) { + this.userButton = this.page.getByRole("button", { name: "User" }); + this.logoutButton = this.page.getByText("Logout"); + } + + async logout() { + await this.userButton.click(); + await this.logoutButton.click(); + } + + async getUserButton() { + return this.userButton; + } +} diff --git a/ui/litellm-dashboard/tests/playwright.config.ts b/ui/litellm-dashboard/tests/playwright.config.ts new file mode 100644 index 0000000000..9a2587bfa9 --- /dev/null +++ b/ui/litellm-dashboard/tests/playwright.config.ts @@ -0,0 +1,76 @@ +import { defineConfig, devices } from "@playwright/test"; +/** + * See https://playwright.dev/docs/test-configuration. + */ + +// import { config } from "dotenv"; +// // import path from "path"; +// config({ path: "./../../../.env.example" }); + +export default defineConfig({ + globalSetup: "utils/globalSetup.ts", + testDir: "./e2e", + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: "html", + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: "http://localhost:4000", + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + }, + + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts b/ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts new file mode 100644 index 0000000000..8641cb5f5d --- /dev/null +++ b/ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts @@ -0,0 +1,437 @@ +import { test, expect, type Page } from '@playwright/test'; + +test.beforeEach(async ({ page }) => { + await page.goto('https://demo.playwright.dev/todomvc'); +}); + +const TODO_ITEMS = [ + 'buy some cheese', + 'feed the cat', + 'book a doctors appointment' +] as const; + +test.describe('New Todo', () => { + test('should allow me to add todo items', async ({ page }) => { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + // Create 1st todo. + await newTodo.fill(TODO_ITEMS[0]); + await newTodo.press('Enter'); + + // Make sure the list only has one todo item. + await expect(page.getByTestId('todo-title')).toHaveText([ + TODO_ITEMS[0] + ]); + + // Create 2nd todo. + await newTodo.fill(TODO_ITEMS[1]); + await newTodo.press('Enter'); + + // Make sure the list now has two todo items. + await expect(page.getByTestId('todo-title')).toHaveText([ + TODO_ITEMS[0], + TODO_ITEMS[1] + ]); + + await checkNumberOfTodosInLocalStorage(page, 2); + }); + + test('should clear text input field when an item is added', async ({ page }) => { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + // Create one todo item. + await newTodo.fill(TODO_ITEMS[0]); + await newTodo.press('Enter'); + + // Check that input is empty. + await expect(newTodo).toBeEmpty(); + await checkNumberOfTodosInLocalStorage(page, 1); + }); + + test('should append new items to the bottom of the list', async ({ page }) => { + // Create 3 items. + await createDefaultTodos(page); + + // create a todo count locator + const todoCount = page.getByTestId('todo-count') + + // Check test using different methods. + await expect(page.getByText('3 items left')).toBeVisible(); + await expect(todoCount).toHaveText('3 items left'); + await expect(todoCount).toContainText('3'); + await expect(todoCount).toHaveText(/3/); + + // Check all items in one call. + await expect(page.getByTestId('todo-title')).toHaveText(TODO_ITEMS); + await checkNumberOfTodosInLocalStorage(page, 3); + }); +}); + +test.describe('Mark all as completed', () => { + test.beforeEach(async ({ page }) => { + await createDefaultTodos(page); + await checkNumberOfTodosInLocalStorage(page, 3); + }); + + test.afterEach(async ({ page }) => { + await checkNumberOfTodosInLocalStorage(page, 3); + }); + + test('should allow me to mark all items as completed', async ({ page }) => { + // Complete all todos. + await page.getByLabel('Mark all as complete').check(); + + // Ensure all todos have 'completed' class. + await expect(page.getByTestId('todo-item')).toHaveClass(['completed', 'completed', 'completed']); + await checkNumberOfCompletedTodosInLocalStorage(page, 3); + }); + + test('should allow me to clear the complete state of all items', async ({ page }) => { + const toggleAll = page.getByLabel('Mark all as complete'); + // Check and then immediately uncheck. + await toggleAll.check(); + await toggleAll.uncheck(); + + // Should be no completed classes. + await expect(page.getByTestId('todo-item')).toHaveClass(['', '', '']); + }); + + test('complete all checkbox should update state when items are completed / cleared', async ({ page }) => { + const toggleAll = page.getByLabel('Mark all as complete'); + await toggleAll.check(); + await expect(toggleAll).toBeChecked(); + await checkNumberOfCompletedTodosInLocalStorage(page, 3); + + // Uncheck first todo. + const firstTodo = page.getByTestId('todo-item').nth(0); + await firstTodo.getByRole('checkbox').uncheck(); + + // Reuse toggleAll locator and make sure its not checked. + await expect(toggleAll).not.toBeChecked(); + + await firstTodo.getByRole('checkbox').check(); + await checkNumberOfCompletedTodosInLocalStorage(page, 3); + + // Assert the toggle all is checked again. + await expect(toggleAll).toBeChecked(); + }); +}); + +test.describe('Item', () => { + + test('should allow me to mark items as complete', async ({ page }) => { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + // Create two items. + for (const item of TODO_ITEMS.slice(0, 2)) { + await newTodo.fill(item); + await newTodo.press('Enter'); + } + + // Check first item. + const firstTodo = page.getByTestId('todo-item').nth(0); + await firstTodo.getByRole('checkbox').check(); + await expect(firstTodo).toHaveClass('completed'); + + // Check second item. + const secondTodo = page.getByTestId('todo-item').nth(1); + await expect(secondTodo).not.toHaveClass('completed'); + await secondTodo.getByRole('checkbox').check(); + + // Assert completed class. + await expect(firstTodo).toHaveClass('completed'); + await expect(secondTodo).toHaveClass('completed'); + }); + + test('should allow me to un-mark items as complete', async ({ page }) => { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + // Create two items. + for (const item of TODO_ITEMS.slice(0, 2)) { + await newTodo.fill(item); + await newTodo.press('Enter'); + } + + const firstTodo = page.getByTestId('todo-item').nth(0); + const secondTodo = page.getByTestId('todo-item').nth(1); + const firstTodoCheckbox = firstTodo.getByRole('checkbox'); + + await firstTodoCheckbox.check(); + await expect(firstTodo).toHaveClass('completed'); + await expect(secondTodo).not.toHaveClass('completed'); + await checkNumberOfCompletedTodosInLocalStorage(page, 1); + + await firstTodoCheckbox.uncheck(); + await expect(firstTodo).not.toHaveClass('completed'); + await expect(secondTodo).not.toHaveClass('completed'); + await checkNumberOfCompletedTodosInLocalStorage(page, 0); + }); + + test('should allow me to edit an item', async ({ page }) => { + await createDefaultTodos(page); + + const todoItems = page.getByTestId('todo-item'); + const secondTodo = todoItems.nth(1); + await secondTodo.dblclick(); + await expect(secondTodo.getByRole('textbox', { name: 'Edit' })).toHaveValue(TODO_ITEMS[1]); + await secondTodo.getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); + await secondTodo.getByRole('textbox', { name: 'Edit' }).press('Enter'); + + // Explicitly assert the new text value. + await expect(todoItems).toHaveText([ + TODO_ITEMS[0], + 'buy some sausages', + TODO_ITEMS[2] + ]); + await checkTodosInLocalStorage(page, 'buy some sausages'); + }); +}); + +test.describe('Editing', () => { + test.beforeEach(async ({ page }) => { + await createDefaultTodos(page); + await checkNumberOfTodosInLocalStorage(page, 3); + }); + + test('should hide other controls when editing', async ({ page }) => { + const todoItem = page.getByTestId('todo-item').nth(1); + await todoItem.dblclick(); + await expect(todoItem.getByRole('checkbox')).not.toBeVisible(); + await expect(todoItem.locator('label', { + hasText: TODO_ITEMS[1], + })).not.toBeVisible(); + await checkNumberOfTodosInLocalStorage(page, 3); + }); + + test('should save edits on blur', async ({ page }) => { + const todoItems = page.getByTestId('todo-item'); + await todoItems.nth(1).dblclick(); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).dispatchEvent('blur'); + + await expect(todoItems).toHaveText([ + TODO_ITEMS[0], + 'buy some sausages', + TODO_ITEMS[2], + ]); + await checkTodosInLocalStorage(page, 'buy some sausages'); + }); + + test('should trim entered text', async ({ page }) => { + const todoItems = page.getByTestId('todo-item'); + await todoItems.nth(1).dblclick(); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill(' buy some sausages '); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Enter'); + + await expect(todoItems).toHaveText([ + TODO_ITEMS[0], + 'buy some sausages', + TODO_ITEMS[2], + ]); + await checkTodosInLocalStorage(page, 'buy some sausages'); + }); + + test('should remove the item if an empty text string was entered', async ({ page }) => { + const todoItems = page.getByTestId('todo-item'); + await todoItems.nth(1).dblclick(); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill(''); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Enter'); + + await expect(todoItems).toHaveText([ + TODO_ITEMS[0], + TODO_ITEMS[2], + ]); + }); + + test('should cancel edits on escape', async ({ page }) => { + const todoItems = page.getByTestId('todo-item'); + await todoItems.nth(1).dblclick(); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); + await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Escape'); + await expect(todoItems).toHaveText(TODO_ITEMS); + }); +}); + +test.describe('Counter', () => { + test('should display the current number of todo items', async ({ page }) => { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + // create a todo count locator + const todoCount = page.getByTestId('todo-count') + + await newTodo.fill(TODO_ITEMS[0]); + await newTodo.press('Enter'); + + await expect(todoCount).toContainText('1'); + + await newTodo.fill(TODO_ITEMS[1]); + await newTodo.press('Enter'); + await expect(todoCount).toContainText('2'); + + await checkNumberOfTodosInLocalStorage(page, 2); + }); +}); + +test.describe('Clear completed button', () => { + test.beforeEach(async ({ page }) => { + await createDefaultTodos(page); + }); + + test('should display the correct text', async ({ page }) => { + await page.locator('.todo-list li .toggle').first().check(); + await expect(page.getByRole('button', { name: 'Clear completed' })).toBeVisible(); + }); + + test('should remove completed items when clicked', async ({ page }) => { + const todoItems = page.getByTestId('todo-item'); + await todoItems.nth(1).getByRole('checkbox').check(); + await page.getByRole('button', { name: 'Clear completed' }).click(); + await expect(todoItems).toHaveCount(2); + await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[2]]); + }); + + test('should be hidden when there are no items that are completed', async ({ page }) => { + await page.locator('.todo-list li .toggle').first().check(); + await page.getByRole('button', { name: 'Clear completed' }).click(); + await expect(page.getByRole('button', { name: 'Clear completed' })).toBeHidden(); + }); +}); + +test.describe('Persistence', () => { + test('should persist its data', async ({ page }) => { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + for (const item of TODO_ITEMS.slice(0, 2)) { + await newTodo.fill(item); + await newTodo.press('Enter'); + } + + const todoItems = page.getByTestId('todo-item'); + const firstTodoCheck = todoItems.nth(0).getByRole('checkbox'); + await firstTodoCheck.check(); + await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[1]]); + await expect(firstTodoCheck).toBeChecked(); + await expect(todoItems).toHaveClass(['completed', '']); + + // Ensure there is 1 completed item. + await checkNumberOfCompletedTodosInLocalStorage(page, 1); + + // Now reload. + await page.reload(); + await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[1]]); + await expect(firstTodoCheck).toBeChecked(); + await expect(todoItems).toHaveClass(['completed', '']); + }); +}); + +test.describe('Routing', () => { + test.beforeEach(async ({ page }) => { + await createDefaultTodos(page); + // make sure the app had a chance to save updated todos in storage + // before navigating to a new view, otherwise the items can get lost :( + // in some frameworks like Durandal + await checkTodosInLocalStorage(page, TODO_ITEMS[0]); + }); + + test('should allow me to display active items', async ({ page }) => { + const todoItem = page.getByTestId('todo-item'); + await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); + + await checkNumberOfCompletedTodosInLocalStorage(page, 1); + await page.getByRole('link', { name: 'Active' }).click(); + await expect(todoItem).toHaveCount(2); + await expect(todoItem).toHaveText([TODO_ITEMS[0], TODO_ITEMS[2]]); + }); + + test('should respect the back button', async ({ page }) => { + const todoItem = page.getByTestId('todo-item'); + await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); + + await checkNumberOfCompletedTodosInLocalStorage(page, 1); + + await test.step('Showing all items', async () => { + await page.getByRole('link', { name: 'All' }).click(); + await expect(todoItem).toHaveCount(3); + }); + + await test.step('Showing active items', async () => { + await page.getByRole('link', { name: 'Active' }).click(); + }); + + await test.step('Showing completed items', async () => { + await page.getByRole('link', { name: 'Completed' }).click(); + }); + + await expect(todoItem).toHaveCount(1); + await page.goBack(); + await expect(todoItem).toHaveCount(2); + await page.goBack(); + await expect(todoItem).toHaveCount(3); + }); + + test('should allow me to display completed items', async ({ page }) => { + await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); + await checkNumberOfCompletedTodosInLocalStorage(page, 1); + await page.getByRole('link', { name: 'Completed' }).click(); + await expect(page.getByTestId('todo-item')).toHaveCount(1); + }); + + test('should allow me to display all items', async ({ page }) => { + await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); + await checkNumberOfCompletedTodosInLocalStorage(page, 1); + await page.getByRole('link', { name: 'Active' }).click(); + await page.getByRole('link', { name: 'Completed' }).click(); + await page.getByRole('link', { name: 'All' }).click(); + await expect(page.getByTestId('todo-item')).toHaveCount(3); + }); + + test('should highlight the currently applied filter', async ({ page }) => { + await expect(page.getByRole('link', { name: 'All' })).toHaveClass('selected'); + + //create locators for active and completed links + const activeLink = page.getByRole('link', { name: 'Active' }); + const completedLink = page.getByRole('link', { name: 'Completed' }); + await activeLink.click(); + + // Page change - active items. + await expect(activeLink).toHaveClass('selected'); + await completedLink.click(); + + // Page change - completed items. + await expect(completedLink).toHaveClass('selected'); + }); +}); + +async function createDefaultTodos(page: Page) { + // create a new todo locator + const newTodo = page.getByPlaceholder('What needs to be done?'); + + for (const item of TODO_ITEMS) { + await newTodo.fill(item); + await newTodo.press('Enter'); + } +} + +async function checkNumberOfTodosInLocalStorage(page: Page, expected: number) { + return await page.waitForFunction(e => { + return JSON.parse(localStorage['react-todos']).length === e; + }, expected); +} + +async function checkNumberOfCompletedTodosInLocalStorage(page: Page, expected: number) { + return await page.waitForFunction(e => { + return JSON.parse(localStorage['react-todos']).filter((todo: any) => todo.completed).length === e; + }, expected); +} + +async function checkTodosInLocalStorage(page: Page, title: string) { + return await page.waitForFunction(t => { + return JSON.parse(localStorage['react-todos']).map((todo: any) => todo.title).includes(t); + }, title); +} diff --git a/ui/litellm-dashboard/tests/utils/globalSetup.ts b/ui/litellm-dashboard/tests/utils/globalSetup.ts new file mode 100644 index 0000000000..2fb0a8f956 --- /dev/null +++ b/ui/litellm-dashboard/tests/utils/globalSetup.ts @@ -0,0 +1,8 @@ +import dotenv from "dotenv"; + +export default async function globalSetup() { + dotenv.config({ + //path should be relative to playwright.config.ts + path: "./../../../.env", + }); +} diff --git a/ui/litellm-dashboard/tests/utils/utils.ts b/ui/litellm-dashboard/tests/utils/utils.ts new file mode 100644 index 0000000000..8261063859 --- /dev/null +++ b/ui/litellm-dashboard/tests/utils/utils.ts @@ -0,0 +1,14 @@ +// import * as dotenv from 'dotenv'; +// import { config } from "dotenv"; +// import path from "path"; +// config({ path: "./../../../../.env.example" }); + +export function loginDetailsSet(): Boolean { + // console.log(process.env.DATABASE_URL); + // console.log(process.env.UI_PASSWORD); + let loginDetailsSet = false; + if (process.env.UI_USERNAME && process.env.UI_PASSWORD) { + loginDetailsSet = true; + } + return loginDetailsSet; +} From 9615bf3ec570abba0625d1a9a6144e0cabf916a2 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Wed, 19 Mar 2025 14:04:25 -0400 Subject: [PATCH 02/14] Started add model test --- .../tests/e2e/add-model.spec.ts | 68 +++++++++++++++++++ ui/litellm-dashboard/tests/e2e/login.spec.ts | 20 ++---- 2 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 ui/litellm-dashboard/tests/e2e/add-model.spec.ts diff --git a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts new file mode 100644 index 0000000000..48b6f84ff2 --- /dev/null +++ b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts @@ -0,0 +1,68 @@ +import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +/* 4644_Test_Adding_a_Model */ +test("Adding a Model as Self", async ({ loginPage, virtualKeysPage, page }) => { + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + await loginPage.goto(); + /* await page.screenshot({ path: "./test-results/4644_Test_Adding_a_Model/go-to-login-page.png" }); */ + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + /* await page.screenshot({ path: "./test-results/4644_Test_Adding_a_Model/dashboard.png" }); */ + + await virtualKeysPage.logout(); + await expect( + page.getByRole("heading", { name: "LiteLLM Login" }) + ).toBeVisible(); + /* await page.screenshot({ path: "./test-results/4644_Test_Adding_a_Model/logout.png" }); */ +}); + +/* +await page.goto('http://localhost:4000/sso/key/generate'); + await page.getByRole('textbox', { name: 'Username:' }).click(); + await page.getByRole('textbox', { name: 'Username:' }).fill('admin'); + await page.getByRole('textbox', { name: 'Password:' }).click(); + await page.getByRole('textbox', { name: 'Password:' }).fill('sk-1234'); + await page.getByRole('button', { name: 'Submit' }).click(); + await page.getByRole('menu').getByText('Models').click(); + await page.getByRole('tab', { name: 'Add Model' }).click(); + await page.getByRole('combobox', { name: '* Provider question-circle :' }).click(); + await page.locator('span').filter({ hasText: 'OpenAI' }).click(); + await page.locator('.ant-select-selection-overflow').click(); + await page.getByTitle('omni-moderation-latest', { exact: true }).locator('div').click(); + await page.locator('div').filter({ hasText: /^Model Mappings$/ }).click(); + await page.getByRole('textbox', { name: 'Type...' }).click(); + await page.locator('#model_mappings').getByText('omni-moderation-latest').click(); + await page.getByRole('textbox', { name: 'Type...' }).dblclick(); + await page.getByRole('textbox', { name: 'Type...' }).press('ControlOrMeta+c'); + await page.getByRole('textbox', { name: '* API Key question-circle :' }).click(); + await page.getByRole('textbox', { name: '* API Key question-circle :' }).fill('sk-1234'); + await page.getByRole('button', { name: 'Add Model' }).click(); + await page.getByRole('tab', { name: 'All Models' }).click(); + await page.getByRole('paragraph').filter({ hasText: 'gpt-4o' }).click(); + await page.locator('pre').filter({ hasText: 'gpt-4o' }).click(); + await page.getByText('omni-moderation-late...').first().click(); + await page.getByText('test-model-name').first().click(); + await page.getByText('omni-moderation-late...').nth(1).click(); + await page.getByRole('row', { name: '0f7f03f... omni-moderation-' }).getByRole('paragraph').nth(1).click(); + await page.getByRole('row', { name: '0f7f03f... omni-moderation-' }).locator('pre').first().click(); + await page.getByText('omni-moderation-2024...').click(); + await page.getByRole('row', { name: 'ed5bd9d... omni-moderation-' }).getByRole('paragraph').nth(1).click(); + await page.getByRole('row', { name: 'ed5bd9d... omni-moderation-' }).locator('pre').first().click(); + await page.getByRole('paragraph').filter({ hasText: 'test-model-name' }).click(); + await page.getByRole('row', { name: 'c0be84c... test-model-name' }).getByRole('paragraph').nth(1).click(); + await page.getByRole('row', { name: 'c0be84c... test-model-name' }).locator('pre').first().click(); + await page.getByText('omni-moderation-late...').nth(2).click(); + await page.getByRole('row', { name: '7f57e27... omni-moderation-' }).getByRole('paragraph').nth(1).click(); + await page.getByRole('button', { name: '7f57e27...' }).click(); + await page.getByRole('button', { name: 'Back to Models' }).click(); + await page.locator('pre').filter({ hasText: 'omni-moderation-late...' }).click(); +*/ diff --git a/ui/litellm-dashboard/tests/e2e/login.spec.ts b/ui/litellm-dashboard/tests/e2e/login.spec.ts index 39057fa0bf..c6956865a4 100644 --- a/ui/litellm-dashboard/tests/e2e/login.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/login.spec.ts @@ -2,14 +2,8 @@ import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; import { test, expect } from "./../fixtures/fixtures"; import { loginDetailsSet } from "./../utils/utils"; -// import { config } from "dotenv"; -// import path from "path"; -// config({ path: "./../../../../.env.example" }); - +/* 4644_Test_Basic_Sign_in_Flow */ test("Login", async ({ loginPage, virtualKeysPage, page }) => { - // Check if there are credentials in local environment file and assign credentials as appropriate. - // Login and verify user is directed to the dashboard. - // Do playwright tests run in order. let username = "admin"; let password = "sk-1234"; if (loginDetailsSet()) { @@ -18,21 +12,15 @@ test("Login", async ({ loginPage, virtualKeysPage, page }) => { password = process.env.UI_PASSWORD as string; } await loginPage.goto(); - await page.screenshot({ path: "./test-results/go-to-login-page.png" }); + /* await page.screenshot({ path: "./test-results/4644_Test_Basic_Sign_in_Flow/go-to-login-page.png" }); */ await loginPage.login(username, password); await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - await page.screenshot({ path: "./test-results/dashboard.png" }); + /* await page.screenshot({ path: "./test-results/4644_Test_Basic_Sign_in_Flow/dashboard.png" }); */ await virtualKeysPage.logout(); await expect( page.getByRole("heading", { name: "LiteLLM Login" }) ).toBeVisible(); - await page.screenshot({ path: "./test-results/logout.png" }); + /* await page.screenshot({ path: "./test-results/4644_Test_Basic_Sign_in_Flow/logout.png" }); */ }); - -/* to dos -1. Add ui link to local env file -2. add login details to env file -3. configure screenshots and add path to screenshots -*/ From 10b0a91ba67264f89dadea1da71a3e794694ea2c Mon Sep 17 00:00:00 2001 From: omnisilica Date: Thu, 20 Mar 2025 19:58:03 -0400 Subject: [PATCH 03/14] Added un-parametized version of 4644_Test_Adding_a_Model --- .../tests/e2e/add-model.spec.ts | 75 +++++++++++-- ui/litellm-dashboard/tests/e2e/login.spec.ts | 7 +- .../tests/fixtures/fixtures.ts | 10 ++ .../page-object-models/dashboard-links.ts | 28 +++++ .../tests/page-object-models/login.page.ts | 2 +- .../tests/page-object-models/models.page.ts | 100 ++++++++++++++++++ .../page-object-models/virtual-keys.page.ts | 2 +- .../tests/playwright.config.ts | 4 +- 8 files changed, 215 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts create mode 100644 ui/litellm-dashboard/tests/page-object-models/models.page.ts diff --git a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts index 48b6f84ff2..1c5578522e 100644 --- a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts @@ -1,9 +1,12 @@ -import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; import { test, expect } from "./../fixtures/fixtures"; import { loginDetailsSet } from "./../utils/utils"; -/* 4644_Test_Adding_a_Model */ -test("Adding a Model as Self", async ({ loginPage, virtualKeysPage, page }) => { +test("4644_Test_Adding_a_Model", async ({ + loginPage, + dashboardLinks, + modelsPage, + page, +}) => { let username = "admin"; let password = "sk-1234"; if (loginDetailsSet()) { @@ -11,18 +14,75 @@ test("Adding a Model as Self", async ({ loginPage, virtualKeysPage, page }) => { username = process.env.UI_USERNAME as string; password = process.env.UI_PASSWORD as string; } + + console.log("1. Navigating to 'Login' page and logging in"); await loginPage.goto(); - /* await page.screenshot({ path: "./test-results/4644_Test_Adding_a_Model/go-to-login-page.png" }); */ + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/00_go-to-login-page.png", + }); await loginPage.login(username, password); await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - /* await page.screenshot({ path: "./test-results/4644_Test_Adding_a_Model/dashboard.png" }); */ + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/01_dashboard.png", + }); - await virtualKeysPage.logout(); + //start + // 2. Navigate to 'Models' page. + console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/02_navigate-to-models-page.png", + }); + // 3. Select 'Add Model' in the header of this page. + console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/03_navigate-to-add-models-tab.png", + }); + // 4. Select OpenAI from 'Provider' dropdown. + console.log("4. Selecting OpenAI from 'Provider' dropdown"); + await modelsPage.getProviderCombobox().click(); + await modelsPage.getProviderComboboxOption("OpenAI").click(); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/04_select-openai-provider.png", + }); + // 5. Select model name from 'LiteLLM Model Name(s)' dropdown, and verify that the selected models appear in the 'LiteLLM Model' column in the 'Model Mappings' section. **Note**: Don't select 'All OpenAI Models (Wildcard)'. + console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); + await modelsPage.getLitellModelNameCombobox().click(); + await modelsPage.getLitellmModelNameComboboxOption("gpt-4o").click(); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/05_select-gpt-4o-model.png", + }); + // 6. Add API Key. + console.log("6. Adding API Key"); + await modelsPage.getAPIKeyInputBox().fill("sk-1234"); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/06_enter-api-key.png", + }); + // 7. Click 'Add Model'. + console.log("7. Clicking 'Add Model'"); + await modelsPage.getAddModelSubmitButton(); //.click(); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/07_add-model.png", + }); + // 8. Navigate to 'All Models' and verify that the models added show up in view with the Public name that was given to them. + console.log("8. Navigating to 'All Models'"); + await modelsPage.getAllModelsTab().click(); + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/08_navigate-to-all-models-tab.png", + }); + //stop + + console.log("9. Logging out"); + await page.pause(); + await dashboardLinks.logout(); await expect( page.getByRole("heading", { name: "LiteLLM Login" }) ).toBeVisible(); - /* await page.screenshot({ path: "./test-results/4644_Test_Adding_a_Model/logout.png" }); */ + await page.screenshot({ + path: "./test-results/4644_Test_Adding_a_Model/09_logout.png", + }); }); /* @@ -34,6 +94,7 @@ await page.goto('http://localhost:4000/sso/key/generate'); await page.getByRole('button', { name: 'Submit' }).click(); await page.getByRole('menu').getByText('Models').click(); await page.getByRole('tab', { name: 'Add Model' }).click(); + await page.getByRole('combobox', { name: '* Provider question-circle :' }).click(); await page.locator('span').filter({ hasText: 'OpenAI' }).click(); await page.locator('.ant-select-selection-overflow').click(); diff --git a/ui/litellm-dashboard/tests/e2e/login.spec.ts b/ui/litellm-dashboard/tests/e2e/login.spec.ts index c6956865a4..8997ba6de3 100644 --- a/ui/litellm-dashboard/tests/e2e/login.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/login.spec.ts @@ -2,8 +2,11 @@ import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; import { test, expect } from "./../fixtures/fixtures"; import { loginDetailsSet } from "./../utils/utils"; -/* 4644_Test_Basic_Sign_in_Flow */ -test("Login", async ({ loginPage, virtualKeysPage, page }) => { +test("4644_Test_Basic_Sign_in_Flow", async ({ + loginPage, + virtualKeysPage, + page, +}) => { let username = "admin"; let password = "sk-1234"; if (loginDetailsSet()) { diff --git a/ui/litellm-dashboard/tests/fixtures/fixtures.ts b/ui/litellm-dashboard/tests/fixtures/fixtures.ts index 557bb2386b..b7342d1153 100644 --- a/ui/litellm-dashboard/tests/fixtures/fixtures.ts +++ b/ui/litellm-dashboard/tests/fixtures/fixtures.ts @@ -1,18 +1,28 @@ +import { DashboardLinks } from "./../page-object-models/dashboard-links"; import { VirtualKeysPage } from "./../page-object-models/virtual-keys.page"; import { test as base } from "@playwright/test"; import { LoginPage } from "../page-object-models/login.page"; +import { ModelsPage } from "../page-object-models/models.page"; type Fixtures = { loginPage: LoginPage; + dashboardLinks: DashboardLinks; virtualKeysPage: VirtualKeysPage; + modelsPage: ModelsPage; }; export const test = base.extend({ loginPage: async ({ page }, use) => { await use(new LoginPage(page)); }, + dashboardLinks: async ({ page }, use) => { + await use(new DashboardLinks(page)); + }, virtualKeysPage: async ({ page }, use) => { await use(new VirtualKeysPage(page)); }, + modelsPage: async ({ page }, use) => { + await use(new ModelsPage(page)); + }, }); export { expect } from "@playwright/test"; diff --git a/ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts b/ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts new file mode 100644 index 0000000000..c9b14852ea --- /dev/null +++ b/ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts @@ -0,0 +1,28 @@ +import { Page, Locator } from "@playwright/test"; + +export class DashboardLinks { + private readonly userButton: Locator; + private readonly logoutButton: Locator; + private readonly modelsPageLink: Locator; + + constructor(private readonly page: Page) { + this.userButton = this.page.getByRole("button", { name: "User" }); + this.logoutButton = this.page.getByText("Logout"); + this.modelsPageLink = this.page.getByRole("menuitem", { + name: "block Models", + }); + } + + async logout() { + await this.userButton.click(); + await this.logoutButton.click(); + } + + getUserButton(): Locator { + return this.userButton; + } + + getModelsPageLink(): Locator { + return this.modelsPageLink; + } +} diff --git a/ui/litellm-dashboard/tests/page-object-models/login.page.ts b/ui/litellm-dashboard/tests/page-object-models/login.page.ts index f0890c05f3..14df36b5e4 100644 --- a/ui/litellm-dashboard/tests/page-object-models/login.page.ts +++ b/ui/litellm-dashboard/tests/page-object-models/login.page.ts @@ -7,7 +7,7 @@ export class LoginPage { private readonly loginSubmit: Locator; //Initialize locators in constructor - constructor(public readonly page: Page) { + constructor(private readonly page: Page) { this.usernameInput = this.page.getByRole("textbox", { name: "Username:" }); this.passwordInput = this.page.getByRole("textbox", { name: "Password:" }); this.loginSubmit = this.page.getByRole("button", { name: "Submit" }); diff --git a/ui/litellm-dashboard/tests/page-object-models/models.page.ts b/ui/litellm-dashboard/tests/page-object-models/models.page.ts new file mode 100644 index 0000000000..afa88e894a --- /dev/null +++ b/ui/litellm-dashboard/tests/page-object-models/models.page.ts @@ -0,0 +1,100 @@ +import { Page, Locator } from "@playwright/test"; + +export class ModelsPage { + // Models Page Tabs + private readonly allModelsTab: Locator; + private readonly addModelTab: Locator; + // All Models Tab Locators + // Add Model Tab Form Locators + private readonly providerCombobox: Locator; + // *private openaiProviderComboboxOption: Locator; + private readonly litellmModelNameCombobox: Locator; + // *private readonly litellmModelNameComboboxOption page.getByTitle('omni-moderation-latest', { exact: true }).locator('div') + /*private readonly modelMappingPublicNameInput: Locator;*/ + private readonly apiKeyInput: Locator; + private readonly addModelSubmitButton: Locator; + + constructor(private readonly page: Page) { + // Models Page Tabs + this.allModelsTab = this.page.getByRole("tab", { name: "All Models" }); + this.addModelTab = this.page.getByRole("tab", { name: "Add Model" }); + // Add Model Tab Form Locators + this.providerCombobox = this.page.getByRole("combobox", { + name: "* Provider question-circle :", + }); + /**this.openaiProviderComboboxOption = this.page + .locator("span") + .filter({ hasText: "OpenAI" });*/ + this.litellmModelNameCombobox = this.page.locator( + ".ant-select-selection-overflow" + ); + /*this.modelMappingPublicNameInput = this.page + .getByRole("row", { name: "omni-moderation-latest omni-" }) + .getByTestId("base-input");*/ + this.apiKeyInput = page.getByRole("textbox", { + name: "* API Key question-circle :", + }); + this.addModelSubmitButton = page.getByRole("button", { name: "Add Model" }); + } + + // 'All Model' Tab // + getAllModelsTab(): Locator { + return this.allModelsTab; + } + + // Parametized Locators + getAllModelsTableCellValue(allModelsTableCellValue: string): Locator { + return this.page.getByRole("cell", { name: allModelsTableCellValue }); //.first(); + } + + // 'Add Model' Tab // + getAddModelTab(): Locator { + return this.addModelTab; + } + + // Parametized Form Locators + getProviderComboboxOption(providerComboboxOption: string): Locator { + return this.page + .locator("span") + .filter({ hasText: providerComboboxOption }); + } + + getLitellmModelNameComboboxOption( + litellmModelNameComboboxOption: string + ): Locator { + return this.page + .getByTitle(litellmModelNameComboboxOption, { exact: true }) + .locator("div"); + } + + getLitellmModelMappingModel(litellmModelMappingModel: string): Locator { + return this.page + .locator("#model_mappings") + .getByText(litellmModelMappingModel); + } + + getLitellmModelMappingModelPublicName( + litellmModelMappingModel: string + ): Locator { + return this.page + .getByRole("row", { name: litellmModelMappingModel }) + .getByTestId("base-input"); + } + + // Non-parametized Form Locators + getProviderCombobox(): Locator { + return this.providerCombobox; + } + + getLitellModelNameCombobox(): Locator { + return this.litellmModelNameCombobox; + } + + getAPIKeyInputBox(): Locator { + return this.apiKeyInput; + } + + getAddModelSubmitButton(): Locator { + return this.addModelSubmitButton; + } +} diff --git a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts index 7b1fb72040..fdad32fb0c 100644 --- a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts +++ b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts @@ -4,7 +4,7 @@ export class VirtualKeysPage { private readonly userButton: Locator; private readonly logoutButton: Locator; - constructor(public readonly page: Page) { + constructor(private readonly page: Page) { this.userButton = this.page.getByRole("button", { name: "User" }); this.logoutButton = this.page.getByText("Logout"); } diff --git a/ui/litellm-dashboard/tests/playwright.config.ts b/ui/litellm-dashboard/tests/playwright.config.ts index 9a2587bfa9..cfd7ed7805 100644 --- a/ui/litellm-dashboard/tests/playwright.config.ts +++ b/ui/litellm-dashboard/tests/playwright.config.ts @@ -35,7 +35,7 @@ export default defineConfig({ name: "chromium", use: { ...devices["Desktop Chrome"] }, }, - + /* { name: "firefox", use: { ...devices["Desktop Firefox"] }, @@ -44,7 +44,7 @@ export default defineConfig({ { name: "webkit", use: { ...devices["Desktop Safari"] }, - }, + },*/ /* Test against mobile viewports. */ // { From ac2eebcd99980eaea6ffc558d6d2c53e8aaf1283 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Thu, 20 Mar 2025 20:35:36 -0400 Subject: [PATCH 04/14] Revert "docs(release_cycle.md): clarify release cycle for stable releases on docs" This reverts commit 647187db118b28c4bea7d7e394e22ec495d3ce38. --- docs/my-website/docs/proxy/release_cycle.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/proxy/release_cycle.md b/docs/my-website/docs/proxy/release_cycle.md index c5782087f2..947a4ae6b3 100644 --- a/docs/my-website/docs/proxy/release_cycle.md +++ b/docs/my-website/docs/proxy/release_cycle.md @@ -4,17 +4,9 @@ Litellm Proxy has the following release cycle: - `v1.x.x-nightly`: These are releases which pass ci/cd. - `v1.x.x.rc`: These are releases which pass ci/cd + [manual review](https://github.com/BerriAI/litellm/discussions/8495#discussioncomment-12180711). -- `v1.x.x:main-stable`: These are releases which pass ci/cd + manual review + 3 days of production testing. +- `v1.x.x` OR `v1.x.x-stable`: These are releases which pass ci/cd + manual review + 3 days of production testing. -In production, we recommend using the latest `v1.x.x:main-stable` release. +In production, we recommend using the latest `v1.x.x` release. -Follow our release notes [here](https://github.com/BerriAI/litellm/releases). - - -## FAQ - -### Is there a release schedule for LiteLLM stable release? - -Stable releases come out every week (typically Sunday) - +Follow our release notes [here](https://github.com/BerriAI/litellm/releases). \ No newline at end of file From e80417f7159b53960fb4f2adbf80a067d351bbaa Mon Sep 17 00:00:00 2001 From: omnisilica Date: Sun, 23 Mar 2025 17:55:52 -0400 Subject: [PATCH 05/14] Refined 4644_Test_Adding_a_Model --- .../tests/data/models-and-providers.json | 12 - .../tests/data/providers-and-models.json | 8 + .../tests/e2e/add-model.spec.ts | 214 ++++++++---------- .../tests/page-object-models/models.page.ts | 32 ++- .../tests/playwright.config.ts | 3 +- 5 files changed, 127 insertions(+), 142 deletions(-) delete mode 100644 ui/litellm-dashboard/tests/data/models-and-providers.json create mode 100644 ui/litellm-dashboard/tests/data/providers-and-models.json diff --git a/ui/litellm-dashboard/tests/data/models-and-providers.json b/ui/litellm-dashboard/tests/data/models-and-providers.json deleted file mode 100644 index 1deb1d58cc..0000000000 --- a/ui/litellm-dashboard/tests/data/models-and-providers.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "openai": [ - "omni-moderation-latest", - "omni-moderation-latest-intents", - "omni-moderation-2024-09-26" - ], - "azure": [ - "azure/gpt-4o-mini-realtime-preview-2024-12-17", - "azure/eu/gpt-4o-mini-realtime-preview-2024-12-17", - "azure/us/gpt-4o-mini-realtime-preview-2024-12-17" - ] -} diff --git a/ui/litellm-dashboard/tests/data/providers-and-models.json b/ui/litellm-dashboard/tests/data/providers-and-models.json new file mode 100644 index 0000000000..a82b2a26d9 --- /dev/null +++ b/ui/litellm-dashboard/tests/data/providers-and-models.json @@ -0,0 +1,8 @@ +{ + "OpenAI": [ + "omni-moderation-latest", + "omni-moderation-latest-intents", + "omni-moderation-2024-09-26" + ], + "Anthropic": ["claude-instant-1", "claude-instant-1.2", "claude-2"] +} diff --git a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts index 1c5578522e..5a46976009 100644 --- a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts @@ -1,129 +1,103 @@ import { test, expect } from "./../fixtures/fixtures"; import { loginDetailsSet } from "./../utils/utils"; +const providersAndModels = JSON.parse( + JSON.stringify(require("./../data/providers-and-models.json")) +); -test("4644_Test_Adding_a_Model", async ({ - loginPage, - dashboardLinks, - modelsPage, - page, -}) => { - let username = "admin"; - let password = "sk-1234"; - if (loginDetailsSet()) { - console.log("Login details exist in .env file."); - username = process.env.UI_USERNAME as string; - password = process.env.UI_PASSWORD as string; - } +providersAndModels["OpenAI"].forEach((model: string) => { + test(`4644_Test_Adding_OpenAI's_${model}_model`, async ({ + loginPage, + dashboardLinks, + modelsPage, + page, + }) => { + console.log(model); - console.log("1. Navigating to 'Login' page and logging in"); - await loginPage.goto(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/00_go-to-login-page.png", - }); + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } - await loginPage.login(username, password); - await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/01_dashboard.png", - }); + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`, + }); - //start - // 2. Navigate to 'Models' page. - console.log("2. Navigating to 'Models' page"); - await dashboardLinks.getModelsPageLink().click(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/02_navigate-to-models-page.png", - }); - // 3. Select 'Add Model' in the header of this page. - console.log("3. Selecting 'Add Model' in the header of 'Models' page"); - await modelsPage.getAddModelTab().click(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/03_navigate-to-add-models-tab.png", - }); - // 4. Select OpenAI from 'Provider' dropdown. - console.log("4. Selecting OpenAI from 'Provider' dropdown"); - await modelsPage.getProviderCombobox().click(); - await modelsPage.getProviderComboboxOption("OpenAI").click(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/04_select-openai-provider.png", - }); - // 5. Select model name from 'LiteLLM Model Name(s)' dropdown, and verify that the selected models appear in the 'LiteLLM Model' column in the 'Model Mappings' section. **Note**: Don't select 'All OpenAI Models (Wildcard)'. - console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); - await modelsPage.getLitellModelNameCombobox().click(); - await modelsPage.getLitellmModelNameComboboxOption("gpt-4o").click(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/05_select-gpt-4o-model.png", - }); - // 6. Add API Key. - console.log("6. Adding API Key"); - await modelsPage.getAPIKeyInputBox().fill("sk-1234"); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/06_enter-api-key.png", - }); - // 7. Click 'Add Model'. - console.log("7. Clicking 'Add Model'"); - await modelsPage.getAddModelSubmitButton(); //.click(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/07_add-model.png", - }); - // 8. Navigate to 'All Models' and verify that the models added show up in view with the Public name that was given to them. - console.log("8. Navigating to 'All Models'"); - await modelsPage.getAllModelsTab().click(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/08_navigate-to-all-models-tab.png", - }); - //stop + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`, + }); - console.log("9. Logging out"); - await page.pause(); - await dashboardLinks.logout(); - await expect( - page.getByRole("heading", { name: "LiteLLM Login" }) - ).toBeVisible(); - await page.screenshot({ - path: "./test-results/4644_Test_Adding_a_Model/09_logout.png", + // console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`, + }); + + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`, + }); + + // console.log("4. Selecting OpenAI from 'Provider' dropdown"); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox("OpenAI"); + await modelsPage.getProviderCombobox().press("Enter"); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`, + }); + + // console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); + console.log(model); + await modelsPage.getLitellModelNameCombobox().click(); + await modelsPage.getLitellModelNameCombobox().fill(model); + await modelsPage.getLitellModelNameCombobox().press("Enter"); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/05_select-${model}-model.png`, + }); + await expect(modelsPage.getLitellmModelMappingModel(model)).toBeVisible(); + + // console.log("6. Adding API Key"); + await modelsPage.getAPIKeyInputBox("OpenAI").click(); + await modelsPage.getAPIKeyInputBox("OpenAI").fill("sk-1234"); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/06_enter-api-key.png`, + }); + + // console.log("7. Clicking 'Add Model'"); + await modelsPage.getAddModelSubmitButton().click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/07_add-model.png`, + }); + + // console.log("8. Navigating to 'All Models'"); + await modelsPage.getAllModelsTab().click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/08_navigate-to-all-models-tab.png`, + }); + await expect( + modelsPage.getAllModelsTableCellValue(`openai logo openai`) + ).toBeVisible(); + await expect( + modelsPage.getAllModelsTableCellValue(model.slice(0, 20) + "...") + ).toBeVisible(); + + // console.log("9. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`, + }); + /* + await dashboardLinks.logout(); + await expect( + page.getByRole("heading", { name: "LiteLLM Login" }) + ).toBeVisible(); + */ }); }); - -/* -await page.goto('http://localhost:4000/sso/key/generate'); - await page.getByRole('textbox', { name: 'Username:' }).click(); - await page.getByRole('textbox', { name: 'Username:' }).fill('admin'); - await page.getByRole('textbox', { name: 'Password:' }).click(); - await page.getByRole('textbox', { name: 'Password:' }).fill('sk-1234'); - await page.getByRole('button', { name: 'Submit' }).click(); - await page.getByRole('menu').getByText('Models').click(); - await page.getByRole('tab', { name: 'Add Model' }).click(); - - await page.getByRole('combobox', { name: '* Provider question-circle :' }).click(); - await page.locator('span').filter({ hasText: 'OpenAI' }).click(); - await page.locator('.ant-select-selection-overflow').click(); - await page.getByTitle('omni-moderation-latest', { exact: true }).locator('div').click(); - await page.locator('div').filter({ hasText: /^Model Mappings$/ }).click(); - await page.getByRole('textbox', { name: 'Type...' }).click(); - await page.locator('#model_mappings').getByText('omni-moderation-latest').click(); - await page.getByRole('textbox', { name: 'Type...' }).dblclick(); - await page.getByRole('textbox', { name: 'Type...' }).press('ControlOrMeta+c'); - await page.getByRole('textbox', { name: '* API Key question-circle :' }).click(); - await page.getByRole('textbox', { name: '* API Key question-circle :' }).fill('sk-1234'); - await page.getByRole('button', { name: 'Add Model' }).click(); - await page.getByRole('tab', { name: 'All Models' }).click(); - await page.getByRole('paragraph').filter({ hasText: 'gpt-4o' }).click(); - await page.locator('pre').filter({ hasText: 'gpt-4o' }).click(); - await page.getByText('omni-moderation-late...').first().click(); - await page.getByText('test-model-name').first().click(); - await page.getByText('omni-moderation-late...').nth(1).click(); - await page.getByRole('row', { name: '0f7f03f... omni-moderation-' }).getByRole('paragraph').nth(1).click(); - await page.getByRole('row', { name: '0f7f03f... omni-moderation-' }).locator('pre').first().click(); - await page.getByText('omni-moderation-2024...').click(); - await page.getByRole('row', { name: 'ed5bd9d... omni-moderation-' }).getByRole('paragraph').nth(1).click(); - await page.getByRole('row', { name: 'ed5bd9d... omni-moderation-' }).locator('pre').first().click(); - await page.getByRole('paragraph').filter({ hasText: 'test-model-name' }).click(); - await page.getByRole('row', { name: 'c0be84c... test-model-name' }).getByRole('paragraph').nth(1).click(); - await page.getByRole('row', { name: 'c0be84c... test-model-name' }).locator('pre').first().click(); - await page.getByText('omni-moderation-late...').nth(2).click(); - await page.getByRole('row', { name: '7f57e27... omni-moderation-' }).getByRole('paragraph').nth(1).click(); - await page.getByRole('button', { name: '7f57e27...' }).click(); - await page.getByRole('button', { name: 'Back to Models' }).click(); - await page.locator('pre').filter({ hasText: 'omni-moderation-late...' }).click(); -*/ diff --git a/ui/litellm-dashboard/tests/page-object-models/models.page.ts b/ui/litellm-dashboard/tests/page-object-models/models.page.ts index afa88e894a..f0c0ae3c13 100644 --- a/ui/litellm-dashboard/tests/page-object-models/models.page.ts +++ b/ui/litellm-dashboard/tests/page-object-models/models.page.ts @@ -25,9 +25,7 @@ export class ModelsPage { /**this.openaiProviderComboboxOption = this.page .locator("span") .filter({ hasText: "OpenAI" });*/ - this.litellmModelNameCombobox = this.page.locator( - ".ant-select-selection-overflow" - ); + this.litellmModelNameCombobox = this.page.locator("#model"); /*this.modelMappingPublicNameInput = this.page .getByRole("row", { name: "omni-moderation-latest omni-" }) .getByTestId("base-input");*/ @@ -44,7 +42,9 @@ export class ModelsPage { // Parametized Locators getAllModelsTableCellValue(allModelsTableCellValue: string): Locator { - return this.page.getByRole("cell", { name: allModelsTableCellValue }); //.first(); + return this.page + .getByRole("cell", { name: allModelsTableCellValue }) + .first(); } // 'Add Model' Tab // @@ -53,18 +53,32 @@ export class ModelsPage { } // Parametized Form Locators - getProviderComboboxOption(providerComboboxOption: string): Locator { - return this.page + /*getProviderComboboxOption(providerComboboxOption: string): Locator { + this.page .locator("span") .filter({ hasText: providerComboboxOption }); + }*/ + + fillProviderComboboxBox(providerComboboxText: string) { + this.page + .getByRole("combobox", { name: "* Provider question-circle :" }) + .fill(providerComboboxText); } - getLitellmModelNameComboboxOption( + getLitellmModelNameCombobox(): Locator { + return this.litellmModelNameCombobox; + } + + /*getLitellmModelNameComboboxOption( litellmModelNameComboboxOption: string ): Locator { return this.page .getByTitle(litellmModelNameComboboxOption, { exact: true }) .locator("div"); + }*/ + + fillLitellmModelNameCombobox(litellmModelNameComboboxOption: string) { + this.page.locator("#model").fill(litellmModelNameComboboxOption); } getLitellmModelMappingModel(litellmModelMappingModel: string): Locator { @@ -90,8 +104,8 @@ export class ModelsPage { return this.litellmModelNameCombobox; } - getAPIKeyInputBox(): Locator { - return this.apiKeyInput; + getAPIKeyInputBox(provider: string): Locator { + return this.page.getByRole("textbox", { name: `* ${provider} API Key :` }); } getAddModelSubmitButton(): Locator { diff --git a/ui/litellm-dashboard/tests/playwright.config.ts b/ui/litellm-dashboard/tests/playwright.config.ts index cfd7ed7805..6f89edee51 100644 --- a/ui/litellm-dashboard/tests/playwright.config.ts +++ b/ui/litellm-dashboard/tests/playwright.config.ts @@ -17,7 +17,8 @@ export default defineConfig({ /* Retry on CI only */ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, + // workers: process.env.CI ? 1 : undefined, + workers: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ From b7a5ac584983e6889985788ffdc05a5519828347 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Wed, 26 Mar 2025 21:36:22 -0400 Subject: [PATCH 06/14] Added Test for creating an API Key --- .../tests/data/providers-and-models.json | 22 +++- .../tests/e2e/add-model.spec.ts | 101 ++++++++++++++++-- .../page-object-models/virtual-keys.page.ts | 48 ++++++++- .../tests/playwright.config.ts | 4 + 4 files changed, 165 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/tests/data/providers-and-models.json b/ui/litellm-dashboard/tests/data/providers-and-models.json index a82b2a26d9..95a1e37e8f 100644 --- a/ui/litellm-dashboard/tests/data/providers-and-models.json +++ b/ui/litellm-dashboard/tests/data/providers-and-models.json @@ -1,8 +1,26 @@ { "OpenAI": [ + "Custom Model Name (Enter below)", + "All OpenAI Models (Wildcard)", "omni-moderation-latest", "omni-moderation-latest-intents", - "omni-moderation-2024-09-26" + "omni-moderation-2024-09-26", + "gpt-4", + "gpt-4o", + "gpt-4o-search-preview-2025-03-11", + "gpt-4o-search-preview", + "gpt-4.5-preview" ], - "Anthropic": ["claude-instant-1", "claude-instant-1.2", "claude-2"] + "Anthropic": [ + "Custom Model Name (Enter below)", + "All Anthropic Models (Wildcard)", + "claude-instant-1", + "claude-instant-1.2", + "claude-2", + "claude-2.1", + "claude-3-haiku-20240307", + "claude-3-5-haiku-20241022", + "claude-3-5-haiku-latest", + "claude-3-opus-latest" + ] } diff --git a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts index 5a46976009..0591d60f7b 100644 --- a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts @@ -3,7 +3,7 @@ import { loginDetailsSet } from "./../utils/utils"; const providersAndModels = JSON.parse( JSON.stringify(require("./../data/providers-and-models.json")) ); - +/* providersAndModels["OpenAI"].forEach((model: string) => { test(`4644_Test_Adding_OpenAI's_${model}_model`, async ({ loginPage, @@ -93,11 +93,98 @@ providersAndModels["OpenAI"].forEach((model: string) => { await page.screenshot({ path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`, }); - /* - await dashboardLinks.logout(); - await expect( - page.getByRole("heading", { name: "LiteLLM Login" }) - ).toBeVisible(); - */ + + // await dashboardLinks.logout(); + // await expect(page.getByRole("heading", { name: "LiteLLM Login" })).toBeVisible(); + + }); +});*/ + +Object.entries(providersAndModels).forEach(([provider, model]) => { + console.log(`${provider}: ${model}`); + test(`4644_Test_the_Correct_Dropdown_Shows_When_Adding_${provider}_Models`, async ({ + loginPage, + dashboardLinks, + modelsPage, + page, + }) => { + let username = "admin"; + let password = "sk-1234"; + const excludeLitellmModelNameDropdownValues = [ + "OpenAI", + "OpenAI-Compatible Endpoints (Together AI, etc.)", + "OpenAI Text Completion", + "OpenAI-Compatible Text Completion Models (Together AI, etc.)", + "Anthropic", + ]; + let litellmModelNameDropdownValues = []; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`, + }); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`, + }); + + console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`, + }); + + console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`, + }); + + console.log(`4. Selecting ${model} from 'Provider' dropdown`); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox(provider); + await modelsPage.getProviderCombobox().press("Enter"); + await page.screenshot({ + path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`, + }); + + //Scrape ant-selection-option and add to list + await modelsPage.getLitellModelNameCombobox().click(); + const litellmModelOptions = await page + .locator(".rc-virtual-list-holder-inner") + .locator(".ant-select-item-option-content") + .all(); + + for (const element of litellmModelOptions) { + //excludeLitellmModelNameDropdownValues + let modelNameDropdownValue = await element.innerText(); + if ( + !excludeLitellmModelNameDropdownValues.includes(modelNameDropdownValue) + ) { + litellmModelNameDropdownValues.push(await element.innerText()); + } + } + litellmModelNameDropdownValues.forEach((element) => { + console.log(element); + expect(providersAndModels[provider].includes(element)).toBeTruthy(); + }); + /*const litellmModelOptions = await page.$$( + ".ant-select-item-option-content" + ); + + const arrOfmodelnames: any[] = []; + + for (const element of litellmModelOptions) { + const text = await element.getAttribute("title"); + // console.log((await element.innerHTML()) + "\n"); + }*/ }); }); diff --git a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts index fdad32fb0c..0f7949f2ed 100644 --- a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts +++ b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts @@ -3,10 +3,28 @@ import type { Page, Locator } from "@playwright/test"; export class VirtualKeysPage { private readonly userButton: Locator; private readonly logoutButton: Locator; + private readonly createNewKeyButton: Locator; + private readonly ownedByYouRadioButton: Locator; + private readonly keyNameInput: Locator; + private readonly modelInput: Locator; + private readonly createKeyButton: Locator; + private readonly copyAPIKeyButton: Locator; constructor(private readonly page: Page) { this.userButton = this.page.getByRole("button", { name: "User" }); this.logoutButton = this.page.getByText("Logout"); + this.createNewKeyButton = this.page.getByRole("button", { + name: "+ Create New Key", + }); + this.ownedByYouRadioButton = this.page.getByRole("radio", { name: "You" }); + this.keyNameInput = this.page.getByTestId("base-input"); + this.modelInput = this.page.locator(".ant-select-selection-overflow"); + this.createKeyButton = this.page.getByRole("button", { + name: "Create Key", + }); + this.copyAPIKeyButton = this.page.getByRole("button", { + name: "Copy API Key", + }); } async logout() { @@ -14,7 +32,35 @@ export class VirtualKeysPage { await this.logoutButton.click(); } - async getUserButton() { + getUserButton(): Locator { return this.userButton; } + + getCreateNewKeyButton(): Locator { + return this.createNewKeyButton; + } + + getVirtualKeysTableCellValue(virtualKeysTableCellValue: string): Locator { + return this.page.getByRole("cell", { name: virtualKeysTableCellValue }); + } + + getOwnedByYouRadioButton(): Locator { + return this.ownedByYouRadioButton; + } + + getKeyNameInput(): Locator { + return this.keyNameInput; + } + + getModelInput(): Locator { + return this.modelInput; + } + + getCreateKeyButton(): Locator { + return this.createKeyButton; + } + + getCopyAPIKeyButton(): Locator { + return this.copyAPIKeyButton; + } } diff --git a/ui/litellm-dashboard/tests/playwright.config.ts b/ui/litellm-dashboard/tests/playwright.config.ts index 6f89edee51..ef0f64237d 100644 --- a/ui/litellm-dashboard/tests/playwright.config.ts +++ b/ui/litellm-dashboard/tests/playwright.config.ts @@ -28,6 +28,10 @@ export default defineConfig({ /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: "on-first-retry", + + contextOptions: { + permissions: ["clipboard-read"], + }, }, /* Configure projects for major browsers */ From 07d6c06f70986d8a813b9f0be04e5f41e173cd30 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Wed, 26 Mar 2025 21:37:37 -0400 Subject: [PATCH 07/14] Added create-api-key.spec.ts to test creating api key --- .../tests/e2e/create-api-key.spec.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts diff --git a/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts b/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts new file mode 100644 index 0000000000..0d5b5b2a1e --- /dev/null +++ b/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts @@ -0,0 +1,99 @@ +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ + loginPage, + dashboardLinks, + virtualKeysPage, + modelsPage, + page, +}) => { + let username = "admin"; + let password = "sk-1234"; + let apiKey = ""; + let apiKeyID = ""; + const keyName = "test-key-name-3"; + + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + /* + console.log("2. Clicking the '+ Create New Key' button"); + await virtualKeysPage.getCreateNewKeyButton().click(); + + console.log("3. Clicking the Owned By You radio button."); + await virtualKeysPage.getOwnedByYouRadioButton().check(); + + console.log("4. Entering a key name in the 'Key Name' input."); + await virtualKeysPage.getKeyNameInput().fill(keyName); + await page.screenshot({ + path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/04_enter-key-name.png`, + }); + + console.log("5. Selecting All Team Models"); + await virtualKeysPage.getModelInput().click(); + await page.getByText("All Team Models").click(); + await page.screenshot({ + path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/05_select-all-team-models.png`, + }); + + console.log("6. Clicking 'Create Key'"); + await virtualKeysPage.getCreateKeyButton().click(); + await page.screenshot({ + path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/06_create-api-key.png`, + }); + + console.log("7. Copying the API key to clipboard"); + await virtualKeysPage.getCopyAPIKeyButton().click(); + apiKey = await page.evaluate(async () => { + return await navigator.clipboard.readText(); + }); + console.log("API Key from clipboard: " + apiKey); + console.log("Sliced API Key from clipboard: " + apiKey.slice(0, 8)); + // await page.pause(); + // await page.getByRole("button", { name: "Copy API Key" }).click(); + + console.log("8. Exiting Modal Window"); + await page + .getByRole("dialog") + .filter({ hasText: "Save your KeyPlease save this" }) + .getByLabel("Close", { exact: true }) + .click(); + await expect( + virtualKeysPage.getVirtualKeysTableCellValue(keyName) + ).toBeVisible(); + await page.screenshot({ + path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/08_checkapi-key-created.png`, + });*/ + + console.log("Delete Generated API key"); + // await page.getByRole("button", { name: apiKey.slice(0, 8) + "..." }).click(); + console.log( + await page + .locator("tr.tremor-TableRow-row.h-8") + .nth(1) + .locator(".tremor-Button-text.text-tremor-default") + .innerText() + ); + apiKeyID = await page + .locator("tr.tremor-TableRow-row.h-8") + .nth(1) + .locator(".tremor-Button-text.text-tremor-default") + .innerText(); + /*.evaluate((element) => { + console.log("element" + element); + });*/ + await page.getByRole("button", { name: apiKeyID }).click(); + await page.getByRole("button", { name: "Delete Key" }).click(); + await page.getByRole("button", { name: "Delete", exact: true }).click(); +}); From a96856b33695b3fcd118c7f63d66cdde5f68ab71 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Thu, 27 Mar 2025 00:57:59 -0400 Subject: [PATCH 08/14] Cleaned up e2e files --- .../tests/e2e/add-model.spec.ts | 205 ++++++++---------- .../tests/e2e/create-api-key.spec.ts | 54 ++--- ui/litellm-dashboard/tests/e2e/login.spec.ts | 6 +- 3 files changed, 103 insertions(+), 162 deletions(-) diff --git a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts index 0591d60f7b..5bedc16213 100644 --- a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts @@ -3,7 +3,7 @@ import { loginDetailsSet } from "./../utils/utils"; const providersAndModels = JSON.parse( JSON.stringify(require("./../data/providers-and-models.json")) ); -/* + providersAndModels["OpenAI"].forEach((model: string) => { test(`4644_Test_Adding_OpenAI's_${model}_model`, async ({ loginPage, @@ -11,97 +11,88 @@ providersAndModels["OpenAI"].forEach((model: string) => { modelsPage, page, }) => { - console.log(model); + const excludeLitellmModelNameDropdownValues = [ + "Custom Model Name (Enter below)", + "All OpenAI Models (Wildcard)", + ]; + if (!excludeLitellmModelNameDropdownValues.includes(model)) { + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } - let username = "admin"; - let password = "sk-1234"; - if (loginDetailsSet()) { - console.log("Login details exist in .env file."); - username = process.env.UI_USERNAME as string; - password = process.env.UI_PASSWORD as string; + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); + + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); + + // console.log("4. Selecting OpenAI from 'Provider' dropdown"); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox("OpenAI"); + await modelsPage.getProviderCombobox().press("Enter"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); + + // console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); + await modelsPage.getLitellModelNameCombobox().click(); + await modelsPage.getLitellModelNameCombobox().fill(model); + await modelsPage.getLitellModelNameCombobox().press("Enter"); + await expect(modelsPage.getLitellmModelMappingModel(model)).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/05_select-${model}-model.png`,}); + + // console.log("6. Adding API Key"); + await modelsPage.getAPIKeyInputBox("OpenAI").click(); + await modelsPage.getAPIKeyInputBox("OpenAI").fill("sk-1234"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/06_enter-api-key.png`,}); + + // console.log("7. Clicking 'Add Model'"); + await modelsPage.getAddModelSubmitButton().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/07_add-model.png`,}); + + // console.log("8. Navigating to 'All Models'"); + if (model.length > 20) { + model = model.slice(0, 20) + "..."; + } + await modelsPage.getAllModelsTab().click(); + + await expect( + modelsPage.getAllModelsTableCellValue(`openai logo openai`) + ).toBeVisible(); + await expect(modelsPage.getAllModelsTableCellValue(model)).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/08_navigate-to-all-models-tab.png`,}); + + // console.log("Clean Up - Delete Model Created"); + const modelID = await page + .locator("tr.tremor-TableRow-row.h-8") + .nth(1) + .locator(".tremor-Button-text.text-tremor-default") + .innerText(); + + await page.getByRole("button", { name: modelID }).click(); + await page.getByRole("button", { name: "Delete Model" }).click(); + await page.getByRole("button", { name: "Delete", exact: true }).click(); + + // console.log("9. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`,}); } - - // console.log("1. Navigating to 'Login' page and logging in"); - await loginPage.goto(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`, - }); - - await loginPage.login(username, password); - await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`, - }); - - // console.log("2. Navigating to 'Models' page"); - await dashboardLinks.getModelsPageLink().click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`, - }); - - // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); - await modelsPage.getAddModelTab().click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`, - }); - - // console.log("4. Selecting OpenAI from 'Provider' dropdown"); - await modelsPage.getProviderCombobox().click(); - modelsPage.fillProviderComboboxBox("OpenAI"); - await modelsPage.getProviderCombobox().press("Enter"); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`, - }); - - // console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); - console.log(model); - await modelsPage.getLitellModelNameCombobox().click(); - await modelsPage.getLitellModelNameCombobox().fill(model); - await modelsPage.getLitellModelNameCombobox().press("Enter"); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/05_select-${model}-model.png`, - }); - await expect(modelsPage.getLitellmModelMappingModel(model)).toBeVisible(); - - // console.log("6. Adding API Key"); - await modelsPage.getAPIKeyInputBox("OpenAI").click(); - await modelsPage.getAPIKeyInputBox("OpenAI").fill("sk-1234"); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/06_enter-api-key.png`, - }); - - // console.log("7. Clicking 'Add Model'"); - await modelsPage.getAddModelSubmitButton().click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/07_add-model.png`, - }); - - // console.log("8. Navigating to 'All Models'"); - await modelsPage.getAllModelsTab().click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/08_navigate-to-all-models-tab.png`, - }); - await expect( - modelsPage.getAllModelsTableCellValue(`openai logo openai`) - ).toBeVisible(); - await expect( - modelsPage.getAllModelsTableCellValue(model.slice(0, 20) + "...") - ).toBeVisible(); - - // console.log("9. Logging out"); - await page.getByRole("link", { name: "LiteLLM Brand" }).click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`, - }); - - // await dashboardLinks.logout(); - // await expect(page.getByRole("heading", { name: "LiteLLM Login" })).toBeVisible(); - }); -});*/ +}); Object.entries(providersAndModels).forEach(([provider, model]) => { - console.log(`${provider}: ${model}`); test(`4644_Test_the_Correct_Dropdown_Shows_When_Adding_${provider}_Models`, async ({ loginPage, dashboardLinks, @@ -124,37 +115,27 @@ Object.entries(providersAndModels).forEach(([provider, model]) => { password = process.env.UI_PASSWORD as string; } - console.log("1. Navigating to 'Login' page and logging in"); + // console.log("1. Navigating to 'Login' page and logging in"); await loginPage.goto(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`, - }); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); await loginPage.login(username, password); await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`, - }); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); - console.log("2. Navigating to 'Models' page"); + // console.log("2. Navigating to 'Models' page"); await dashboardLinks.getModelsPageLink().click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`, - }); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); - console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); await modelsPage.getAddModelTab().click(); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`, - }); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); - console.log(`4. Selecting ${model} from 'Provider' dropdown`); + // console.log(`4. Selecting ${model} from 'Provider' dropdown`); await modelsPage.getProviderCombobox().click(); modelsPage.fillProviderComboboxBox(provider); await modelsPage.getProviderCombobox().press("Enter"); - await page.screenshot({ - path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`, - }); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); //Scrape ant-selection-option and add to list await modelsPage.getLitellModelNameCombobox().click(); @@ -164,7 +145,6 @@ Object.entries(providersAndModels).forEach(([provider, model]) => { .all(); for (const element of litellmModelOptions) { - //excludeLitellmModelNameDropdownValues let modelNameDropdownValue = await element.innerText(); if ( !excludeLitellmModelNameDropdownValues.includes(modelNameDropdownValue) @@ -173,18 +153,7 @@ Object.entries(providersAndModels).forEach(([provider, model]) => { } } litellmModelNameDropdownValues.forEach((element) => { - console.log(element); expect(providersAndModels[provider].includes(element)).toBeTruthy(); }); - /*const litellmModelOptions = await page.$$( - ".ant-select-item-option-content" - ); - - const arrOfmodelnames: any[] = []; - - for (const element of litellmModelOptions) { - const text = await element.getAttribute("title"); - // console.log((await element.innerHTML()) + "\n"); - }*/ }); }); diff --git a/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts b/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts index 0d5b5b2a1e..3d942a496d 100644 --- a/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts @@ -3,9 +3,7 @@ import { loginDetailsSet } from "./../utils/utils"; test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ loginPage, - dashboardLinks, virtualKeysPage, - modelsPage, page, }) => { let username = "admin"; @@ -15,55 +13,44 @@ test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ const keyName = "test-key-name-3"; if (loginDetailsSet()) { - console.log("Login details exist in .env file."); username = process.env.UI_USERNAME as string; password = process.env.UI_PASSWORD as string; } - console.log("1. Navigating to 'Login' page and logging in"); + // console.log("1. Navigating to 'Login' page and logging in"); await loginPage.goto(); // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); await loginPage.login(username, password); await expect(page.getByRole("button", { name: "User" })).toBeVisible(); // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); - /* - console.log("2. Clicking the '+ Create New Key' button"); + + // console.log("2. Clicking the '+ Create New Key' button"); await virtualKeysPage.getCreateNewKeyButton().click(); - console.log("3. Clicking the Owned By You radio button."); + // console.log("3. Clicking the Owned By You radio button."); await virtualKeysPage.getOwnedByYouRadioButton().check(); - console.log("4. Entering a key name in the 'Key Name' input."); + // console.log("4. Entering a key name in the 'Key Name' input."); await virtualKeysPage.getKeyNameInput().fill(keyName); - await page.screenshot({ - path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/04_enter-key-name.png`, - }); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/04_enter-key-name.png`,}); - console.log("5. Selecting All Team Models"); + // console.log("5. Selecting All Team Models"); await virtualKeysPage.getModelInput().click(); await page.getByText("All Team Models").click(); - await page.screenshot({ - path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/05_select-all-team-models.png`, - }); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/05_select-all-team-models.png`,}); - console.log("6. Clicking 'Create Key'"); + // console.log("6. Clicking 'Create Key'"); await virtualKeysPage.getCreateKeyButton().click(); - await page.screenshot({ - path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/06_create-api-key.png`, - }); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/06_create-api-key.png`,}); - console.log("7. Copying the API key to clipboard"); + // console.log("7. Copying the API key to clipboard"); await virtualKeysPage.getCopyAPIKeyButton().click(); apiKey = await page.evaluate(async () => { return await navigator.clipboard.readText(); }); - console.log("API Key from clipboard: " + apiKey); - console.log("Sliced API Key from clipboard: " + apiKey.slice(0, 8)); - // await page.pause(); - // await page.getByRole("button", { name: "Copy API Key" }).click(); - console.log("8. Exiting Modal Window"); + // console.log("8. Exiting Modal Window"); await page .getByRole("dialog") .filter({ hasText: "Save your KeyPlease save this" }) @@ -72,27 +59,12 @@ test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ await expect( virtualKeysPage.getVirtualKeysTableCellValue(keyName) ).toBeVisible(); - await page.screenshot({ - path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/08_checkapi-key-created.png`, - });*/ + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/08_check-api-key-created.png`,}); - console.log("Delete Generated API key"); - // await page.getByRole("button", { name: apiKey.slice(0, 8) + "..." }).click(); - console.log( - await page - .locator("tr.tremor-TableRow-row.h-8") - .nth(1) - .locator(".tremor-Button-text.text-tremor-default") - .innerText() - ); apiKeyID = await page .locator("tr.tremor-TableRow-row.h-8") - .nth(1) .locator(".tremor-Button-text.text-tremor-default") .innerText(); - /*.evaluate((element) => { - console.log("element" + element); - });*/ await page.getByRole("button", { name: apiKeyID }).click(); await page.getByRole("button", { name: "Delete Key" }).click(); await page.getByRole("button", { name: "Delete", exact: true }).click(); diff --git a/ui/litellm-dashboard/tests/e2e/login.spec.ts b/ui/litellm-dashboard/tests/e2e/login.spec.ts index 8997ba6de3..a7de95d575 100644 --- a/ui/litellm-dashboard/tests/e2e/login.spec.ts +++ b/ui/litellm-dashboard/tests/e2e/login.spec.ts @@ -15,15 +15,15 @@ test("4644_Test_Basic_Sign_in_Flow", async ({ password = process.env.UI_PASSWORD as string; } await loginPage.goto(); - /* await page.screenshot({ path: "./test-results/4644_Test_Basic_Sign_in_Flow/go-to-login-page.png" }); */ + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/navigate-to-login-page.png",}); await loginPage.login(username, password); await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - /* await page.screenshot({ path: "./test-results/4644_Test_Basic_Sign_in_Flow/dashboard.png" }); */ + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/dashboard.png",}); await virtualKeysPage.logout(); await expect( page.getByRole("heading", { name: "LiteLLM Login" }) ).toBeVisible(); - /* await page.screenshot({ path: "./test-results/4644_Test_Basic_Sign_in_Flow/logout.png" }); */ + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/logout.png",}); }); From 68e82ecba8e4da7a43011d405a8c28f9b4c60da2 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Mon, 31 Mar 2025 23:16:04 -0400 Subject: [PATCH 09/14] Transferred e2e from 'litellm/ui/litellm-dashboard/src/' to 'litellm/tests/proxy_admin_ui_tests' --- .../e2e_ui_tests/login_to_ui.spec.ts | 34 +- .../e2e_ui_tests/view_internal_user.spec.ts | 76 ++- tests/proxy_admin_ui_tests/package-lock.json | 30 +- tests/proxy_admin_ui_tests/package.json | 5 +- .../proxy_admin_ui_tests/playwright.config.ts | 46 +- .../tests/data/providers-and-models.json | 26 -- .../tests/e2e/add-model.spec.ts | 159 ------- .../tests/e2e/create-api-key.spec.ts | 71 --- .../tests/e2e/example.spec.ts | 18 - ui/litellm-dashboard/tests/e2e/login.spec.ts | 29 -- .../tests/fixtures/fixtures.ts | 28 -- ui/litellm-dashboard/tests/package-lock.json | 105 ----- ui/litellm-dashboard/tests/package.json | 17 - .../page-object-models/dashboard-links.ts | 28 -- .../tests/page-object-models/login.page.ts | 27 -- .../tests/page-object-models/models.page.ts | 114 ----- .../page-object-models/virtual-keys.page.ts | 66 --- .../tests/playwright.config.ts | 81 ---- .../tests-examples/demo-todo-app.spec.ts | 437 ------------------ .../tests/utils/globalSetup.ts | 8 - ui/litellm-dashboard/tests/utils/utils.ts | 14 - 21 files changed, 137 insertions(+), 1282 deletions(-) delete mode 100644 ui/litellm-dashboard/tests/data/providers-and-models.json delete mode 100644 ui/litellm-dashboard/tests/e2e/add-model.spec.ts delete mode 100644 ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts delete mode 100644 ui/litellm-dashboard/tests/e2e/example.spec.ts delete mode 100644 ui/litellm-dashboard/tests/e2e/login.spec.ts delete mode 100644 ui/litellm-dashboard/tests/fixtures/fixtures.ts delete mode 100644 ui/litellm-dashboard/tests/package-lock.json delete mode 100644 ui/litellm-dashboard/tests/package.json delete mode 100644 ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts delete mode 100644 ui/litellm-dashboard/tests/page-object-models/login.page.ts delete mode 100644 ui/litellm-dashboard/tests/page-object-models/models.page.ts delete mode 100644 ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts delete mode 100644 ui/litellm-dashboard/tests/playwright.config.ts delete mode 100644 ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts delete mode 100644 ui/litellm-dashboard/tests/utils/globalSetup.ts delete mode 100644 ui/litellm-dashboard/tests/utils/utils.ts diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts index f5148bed16..4a32c39c6f 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts @@ -6,17 +6,17 @@ Basic UI Test Click on all the tabs ensure nothing is broken */ -import { test, expect } from '@playwright/test'; +import { test, expect } from "@playwright/test"; -test('admin login test', async ({ page }) => { +test("admin login test", async ({ page }) => { // Go to the specified URL - await page.goto('http://localhost:4000/ui'); + await page.goto("http://localhost:4000/ui"); // Enter "admin" in the username input field - await page.fill('input[name="username"]', 'admin'); + await page.fill('input[name="username"]', "admin"); // Enter "gm" in the password input field - await page.fill('input[name="password"]', 'gm'); + await page.fill('input[name="password"]', "sk-1234"); // Optionally, you can add an assertion to verify the login button is enabled const loginButton = page.locator('input[type="submit"]'); @@ -25,20 +25,22 @@ test('admin login test', async ({ page }) => { // Optionally, you can click the login button to submit the form await loginButton.click(); const tabs = [ - 'Virtual Keys', - 'Test Key', - 'Models', - 'Usage', - 'Teams', - 'Internal User', - 'Settings', - 'Experimental', - 'API Reference', - 'Model Hub' + "Virtual Keys", + "Test Key", + "Models", + "Usage", + "Teams", + "Internal User", + "Settings", + "Experimental", + "API Reference", + "Model Hub", ]; for (const tab of tabs) { - const tabElement = page.locator('span.ant-menu-title-content', { hasText: tab }); + const tabElement = page.locator("span.ant-menu-title-content", { + hasText: tab, + }); await tabElement.click(); } }); diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts index 4d27a4a7ce..de2c8a5720 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts @@ -2,17 +2,17 @@ Test view internal user page */ -import { test, expect } from '@playwright/test'; +import { test, expect } from "@playwright/test"; -test('view internal user page', async ({ page }) => { +test("view internal user page", async ({ page }) => { // Go to the specified URL - await page.goto('http://localhost:4000/ui'); + await page.goto("http://localhost:4000/ui"); // Enter "admin" in the username input field - await page.fill('input[name="username"]', 'admin'); + await page.fill('input[name="username"]', "admin"); // Enter "gm" in the password input field - await page.fill('input[name="password"]', 'gm'); + await page.fill('input[name="password"]', "sk-1234"); // Optionally, you can add an assertion to verify the login button is enabled const loginButton = page.locator('input[type="submit"]'); @@ -21,26 +21,70 @@ test('view internal user page', async ({ page }) => { // Optionally, you can click the login button to submit the form await loginButton.click(); - const tabElement = page.locator('span.ant-menu-title-content', { hasText: 'Internal User' }); + const tabElement = page.locator("span.ant-menu-title-content", { + hasText: "Internal User", + }); await tabElement.click(); - // try to click on button + // try to click on button // // wait 1-2 seconds - await page.waitForTimeout(10000); + // await page.waitForTimeout(10000); - // Test all expected fields are present - // number of keys owned by user - const keysBadges = page.locator('p.tremor-Badge-text.text-sm.whitespace-nowrap', { hasText: 'Keys' }); - const keysCountArray = await keysBadges.evaluateAll(elements => elements.map(el => parseInt(el.textContent.split(' ')[0], 10))); + // Test all expected fields are present + // number of keys owned by user + const keysBadges = await page + .locator("p.tremor-Badge-text.text-sm.whitespace-nowrap", { + hasText: "Keys", + }) + .all(); - const hasNonZeroKeys = keysCountArray.some(count => count > 0); + await page.pause(); + + /* + const keysCountArray = await keysBadges.evaluateAll((elements) => { + elements.map((el) => { + console.log(el); + parseInt(el.textContent.split(" ")[0], 10); + }); + }); + */ + + let keysCountArray: number[] = []; + + for (const element of keysBadges) { + keysCountArray.push( + parseInt((await element.innerText()).split(" ")[0], 10) + ); + } + + const hasNonZeroKeys = keysCountArray.some((count) => count > 0); expect(hasNonZeroKeys).toBe(true); // test pagination - const prevButton = page.locator('button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed', { hasText: 'Previous' }); + const prevButton = page.locator( + "button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed", + { hasText: "Previous" } + ); await expect(prevButton).toBeDisabled(); - const nextButton = page.locator('button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed', { hasText: 'Next' }); - await expect(nextButton).toBeEnabled(); + let paginationText = await page + .locator(".flex.items-center.space-x-2") + .locator(".text-sm.text-gray-700") + .innerText(); + + let paginationTextContents = paginationText.split(" "); + + let totalPages = parseInt( + paginationTextContents[paginationTextContents.length - 1], + 10 + ); + + if (totalPages > 1) { + const nextButton = page.locator( + "button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed", + { hasText: "Next" } + ); + await expect(nextButton).toBeEnabled(); + } }); diff --git a/tests/proxy_admin_ui_tests/package-lock.json b/tests/proxy_admin_ui_tests/package-lock.json index 3152ee9bfa..0b597aed9c 100644 --- a/tests/proxy_admin_ui_tests/package-lock.json +++ b/tests/proxy_admin_ui_tests/package-lock.json @@ -8,9 +8,12 @@ "name": "proxy_admin_ui_tests", "version": "1.0.0", "license": "ISC", + "dependencies": { + "dotenv": "^16.4.7" + }, "devDependencies": { "@playwright/test": "^1.47.2", - "@types/node": "^22.5.5" + "@types/node": "^22.13.14" } }, "node_modules/@playwright/test": { @@ -29,12 +32,23 @@ } }, "node_modules/@types/node": { - "version": "22.5.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz", - "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==", + "version": "22.13.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.14.tgz", + "integrity": "sha512-Zs/Ollc1SJ8nKUAgc7ivOEdIBM8JAKgrqqUYi2J997JuKO7/tpQC+WCetQ1sypiKCQWHdvdg9wBNpUPEWZae7w==", "dev": true, "dependencies": { - "undici-types": "~6.19.2" + "undici-types": "~6.20.0" + } + }, + "node_modules/dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, "node_modules/fsevents": { @@ -82,9 +96,9 @@ } }, "node_modules/undici-types": { - "version": "6.19.8", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", "dev": true } } diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index 20dfed7a8a..823f65b8df 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -9,6 +9,9 @@ "license": "ISC", "devDependencies": { "@playwright/test": "^1.47.2", - "@types/node": "^22.5.5" + "@types/node": "^22.13.14" + }, + "dependencies": { + "dotenv": "^16.4.7" } } diff --git a/tests/proxy_admin_ui_tests/playwright.config.ts b/tests/proxy_admin_ui_tests/playwright.config.ts index 3be77a319e..d9e0a67453 100644 --- a/tests/proxy_admin_ui_tests/playwright.config.ts +++ b/tests/proxy_admin_ui_tests/playwright.config.ts @@ -1,4 +1,4 @@ -import { defineConfig, devices } from '@playwright/test'; +import { defineConfig, devices } from "@playwright/test"; /** * Read environment variables from file. @@ -12,9 +12,15 @@ import { defineConfig, devices } from '@playwright/test'; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - testDir: './e2e_ui_tests', - testIgnore: ['**/tests/pass_through_tests/**', '../pass_through_tests/**/*'], - testMatch: '**/*.spec.ts', // Only run files ending in .spec.ts + globalSetup: "utils/globalSetup.ts", + testDir: "./e2e_ui_tests", + testIgnore: [ + "**/tests/pass_through_tests/**", + "../pass_through_tests/**/*", + "./e2e_ui_tests/login_to_ui.spec.ts", + "./e2e_ui_tests/view_internal_user.spec.ts", + ], + testMatch: "**/*.spec.ts", // Only run files ending in .spec.ts /* Run tests in files in parallel */ fullyParallel: true, /* Fail the build on CI if you accidentally left test.only in the source code. */ @@ -22,33 +28,47 @@ export default defineConfig({ /* Retry on CI only */ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, + // workers: process.env.CI ? 1 : undefined, + workers: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ // baseURL: 'http://127.0.0.1:3000', + baseURL: "http://localhost:4000", /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + trace: "on-first-retry", }, /* Configure projects for major browsers */ projects: [ { - name: 'chromium', - use: { ...devices['Desktop Chrome'] }, + name: "chromium", + use: { + ...devices["Desktop Chrome"], + contextOptions: { + permissions: ["clipboard-read", "clipboard-write"], + }, + }, }, { - name: 'firefox', - use: { ...devices['Desktop Firefox'] }, + name: "firefox", + use: { + ...devices["Desktop Firefox"], + }, }, { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, + name: "webkit", + use: { + ...devices["Desktop Safari"], + contextOptions: { + permissions: ["clipboard-read"], + }, + }, }, /* Test against mobile viewports. */ diff --git a/ui/litellm-dashboard/tests/data/providers-and-models.json b/ui/litellm-dashboard/tests/data/providers-and-models.json deleted file mode 100644 index 95a1e37e8f..0000000000 --- a/ui/litellm-dashboard/tests/data/providers-and-models.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "OpenAI": [ - "Custom Model Name (Enter below)", - "All OpenAI Models (Wildcard)", - "omni-moderation-latest", - "omni-moderation-latest-intents", - "omni-moderation-2024-09-26", - "gpt-4", - "gpt-4o", - "gpt-4o-search-preview-2025-03-11", - "gpt-4o-search-preview", - "gpt-4.5-preview" - ], - "Anthropic": [ - "Custom Model Name (Enter below)", - "All Anthropic Models (Wildcard)", - "claude-instant-1", - "claude-instant-1.2", - "claude-2", - "claude-2.1", - "claude-3-haiku-20240307", - "claude-3-5-haiku-20241022", - "claude-3-5-haiku-latest", - "claude-3-opus-latest" - ] -} diff --git a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts b/ui/litellm-dashboard/tests/e2e/add-model.spec.ts deleted file mode 100644 index 5bedc16213..0000000000 --- a/ui/litellm-dashboard/tests/e2e/add-model.spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { test, expect } from "./../fixtures/fixtures"; -import { loginDetailsSet } from "./../utils/utils"; -const providersAndModels = JSON.parse( - JSON.stringify(require("./../data/providers-and-models.json")) -); - -providersAndModels["OpenAI"].forEach((model: string) => { - test(`4644_Test_Adding_OpenAI's_${model}_model`, async ({ - loginPage, - dashboardLinks, - modelsPage, - page, - }) => { - const excludeLitellmModelNameDropdownValues = [ - "Custom Model Name (Enter below)", - "All OpenAI Models (Wildcard)", - ]; - if (!excludeLitellmModelNameDropdownValues.includes(model)) { - let username = "admin"; - let password = "sk-1234"; - if (loginDetailsSet()) { - console.log("Login details exist in .env file."); - username = process.env.UI_USERNAME as string; - password = process.env.UI_PASSWORD as string; - } - - // console.log("1. Navigating to 'Login' page and logging in"); - await loginPage.goto(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); - - await loginPage.login(username, password); - await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); - - // console.log("2. Navigating to 'Models' page"); - await dashboardLinks.getModelsPageLink().click(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); - - // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); - await modelsPage.getAddModelTab().click(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); - - // console.log("4. Selecting OpenAI from 'Provider' dropdown"); - await modelsPage.getProviderCombobox().click(); - modelsPage.fillProviderComboboxBox("OpenAI"); - await modelsPage.getProviderCombobox().press("Enter"); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); - - // console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); - await modelsPage.getLitellModelNameCombobox().click(); - await modelsPage.getLitellModelNameCombobox().fill(model); - await modelsPage.getLitellModelNameCombobox().press("Enter"); - await expect(modelsPage.getLitellmModelMappingModel(model)).toBeVisible(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/05_select-${model}-model.png`,}); - - // console.log("6. Adding API Key"); - await modelsPage.getAPIKeyInputBox("OpenAI").click(); - await modelsPage.getAPIKeyInputBox("OpenAI").fill("sk-1234"); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/06_enter-api-key.png`,}); - - // console.log("7. Clicking 'Add Model'"); - await modelsPage.getAddModelSubmitButton().click(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/07_add-model.png`,}); - - // console.log("8. Navigating to 'All Models'"); - if (model.length > 20) { - model = model.slice(0, 20) + "..."; - } - await modelsPage.getAllModelsTab().click(); - - await expect( - modelsPage.getAllModelsTableCellValue(`openai logo openai`) - ).toBeVisible(); - await expect(modelsPage.getAllModelsTableCellValue(model)).toBeVisible(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/08_navigate-to-all-models-tab.png`,}); - - // console.log("Clean Up - Delete Model Created"); - const modelID = await page - .locator("tr.tremor-TableRow-row.h-8") - .nth(1) - .locator(".tremor-Button-text.text-tremor-default") - .innerText(); - - await page.getByRole("button", { name: modelID }).click(); - await page.getByRole("button", { name: "Delete Model" }).click(); - await page.getByRole("button", { name: "Delete", exact: true }).click(); - - // console.log("9. Logging out"); - await page.getByRole("link", { name: "LiteLLM Brand" }).click(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`,}); - } - }); -}); - -Object.entries(providersAndModels).forEach(([provider, model]) => { - test(`4644_Test_the_Correct_Dropdown_Shows_When_Adding_${provider}_Models`, async ({ - loginPage, - dashboardLinks, - modelsPage, - page, - }) => { - let username = "admin"; - let password = "sk-1234"; - const excludeLitellmModelNameDropdownValues = [ - "OpenAI", - "OpenAI-Compatible Endpoints (Together AI, etc.)", - "OpenAI Text Completion", - "OpenAI-Compatible Text Completion Models (Together AI, etc.)", - "Anthropic", - ]; - let litellmModelNameDropdownValues = []; - if (loginDetailsSet()) { - console.log("Login details exist in .env file."); - username = process.env.UI_USERNAME as string; - password = process.env.UI_PASSWORD as string; - } - - // console.log("1. Navigating to 'Login' page and logging in"); - await loginPage.goto(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); - - await loginPage.login(username, password); - await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); - - // console.log("2. Navigating to 'Models' page"); - await dashboardLinks.getModelsPageLink().click(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); - - // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); - await modelsPage.getAddModelTab().click(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); - - // console.log(`4. Selecting ${model} from 'Provider' dropdown`); - await modelsPage.getProviderCombobox().click(); - modelsPage.fillProviderComboboxBox(provider); - await modelsPage.getProviderCombobox().press("Enter"); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); - - //Scrape ant-selection-option and add to list - await modelsPage.getLitellModelNameCombobox().click(); - const litellmModelOptions = await page - .locator(".rc-virtual-list-holder-inner") - .locator(".ant-select-item-option-content") - .all(); - - for (const element of litellmModelOptions) { - let modelNameDropdownValue = await element.innerText(); - if ( - !excludeLitellmModelNameDropdownValues.includes(modelNameDropdownValue) - ) { - litellmModelNameDropdownValues.push(await element.innerText()); - } - } - litellmModelNameDropdownValues.forEach((element) => { - expect(providersAndModels[provider].includes(element)).toBeTruthy(); - }); - }); -}); diff --git a/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts b/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts deleted file mode 100644 index 3d942a496d..0000000000 --- a/ui/litellm-dashboard/tests/e2e/create-api-key.spec.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { test, expect } from "./../fixtures/fixtures"; -import { loginDetailsSet } from "./../utils/utils"; - -test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ - loginPage, - virtualKeysPage, - page, -}) => { - let username = "admin"; - let password = "sk-1234"; - let apiKey = ""; - let apiKeyID = ""; - const keyName = "test-key-name-3"; - - if (loginDetailsSet()) { - username = process.env.UI_USERNAME as string; - password = process.env.UI_PASSWORD as string; - } - - // console.log("1. Navigating to 'Login' page and logging in"); - await loginPage.goto(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); - - await loginPage.login(username, password); - await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); - - // console.log("2. Clicking the '+ Create New Key' button"); - await virtualKeysPage.getCreateNewKeyButton().click(); - - // console.log("3. Clicking the Owned By You radio button."); - await virtualKeysPage.getOwnedByYouRadioButton().check(); - - // console.log("4. Entering a key name in the 'Key Name' input."); - await virtualKeysPage.getKeyNameInput().fill(keyName); - // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/04_enter-key-name.png`,}); - - // console.log("5. Selecting All Team Models"); - await virtualKeysPage.getModelInput().click(); - await page.getByText("All Team Models").click(); - // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/05_select-all-team-models.png`,}); - - // console.log("6. Clicking 'Create Key'"); - await virtualKeysPage.getCreateKeyButton().click(); - // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/06_create-api-key.png`,}); - - // console.log("7. Copying the API key to clipboard"); - await virtualKeysPage.getCopyAPIKeyButton().click(); - apiKey = await page.evaluate(async () => { - return await navigator.clipboard.readText(); - }); - - // console.log("8. Exiting Modal Window"); - await page - .getByRole("dialog") - .filter({ hasText: "Save your KeyPlease save this" }) - .getByLabel("Close", { exact: true }) - .click(); - await expect( - virtualKeysPage.getVirtualKeysTableCellValue(keyName) - ).toBeVisible(); - // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/08_check-api-key-created.png`,}); - - apiKeyID = await page - .locator("tr.tremor-TableRow-row.h-8") - .locator(".tremor-Button-text.text-tremor-default") - .innerText(); - await page.getByRole("button", { name: apiKeyID }).click(); - await page.getByRole("button", { name: "Delete Key" }).click(); - await page.getByRole("button", { name: "Delete", exact: true }).click(); -}); diff --git a/ui/litellm-dashboard/tests/e2e/example.spec.ts b/ui/litellm-dashboard/tests/e2e/example.spec.ts deleted file mode 100644 index 54a906a4e8..0000000000 --- a/ui/litellm-dashboard/tests/e2e/example.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { test, expect } from '@playwright/test'; - -test('has title', async ({ page }) => { - await page.goto('https://playwright.dev/'); - - // Expect a title "to contain" a substring. - await expect(page).toHaveTitle(/Playwright/); -}); - -test('get started link', async ({ page }) => { - await page.goto('https://playwright.dev/'); - - // Click the get started link. - await page.getByRole('link', { name: 'Get started' }).click(); - - // Expects page to have a heading with the name of Installation. - await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible(); -}); diff --git a/ui/litellm-dashboard/tests/e2e/login.spec.ts b/ui/litellm-dashboard/tests/e2e/login.spec.ts deleted file mode 100644 index a7de95d575..0000000000 --- a/ui/litellm-dashboard/tests/e2e/login.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; -import { test, expect } from "./../fixtures/fixtures"; -import { loginDetailsSet } from "./../utils/utils"; - -test("4644_Test_Basic_Sign_in_Flow", async ({ - loginPage, - virtualKeysPage, - page, -}) => { - let username = "admin"; - let password = "sk-1234"; - if (loginDetailsSet()) { - console.log("Login details exist in .env file."); - username = process.env.UI_USERNAME as string; - password = process.env.UI_PASSWORD as string; - } - await loginPage.goto(); - // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/navigate-to-login-page.png",}); - - await loginPage.login(username, password); - await expect(page.getByRole("button", { name: "User" })).toBeVisible(); - // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/dashboard.png",}); - - await virtualKeysPage.logout(); - await expect( - page.getByRole("heading", { name: "LiteLLM Login" }) - ).toBeVisible(); - // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/logout.png",}); -}); diff --git a/ui/litellm-dashboard/tests/fixtures/fixtures.ts b/ui/litellm-dashboard/tests/fixtures/fixtures.ts deleted file mode 100644 index b7342d1153..0000000000 --- a/ui/litellm-dashboard/tests/fixtures/fixtures.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { DashboardLinks } from "./../page-object-models/dashboard-links"; -import { VirtualKeysPage } from "./../page-object-models/virtual-keys.page"; -import { test as base } from "@playwright/test"; -import { LoginPage } from "../page-object-models/login.page"; -import { ModelsPage } from "../page-object-models/models.page"; - -type Fixtures = { - loginPage: LoginPage; - dashboardLinks: DashboardLinks; - virtualKeysPage: VirtualKeysPage; - modelsPage: ModelsPage; -}; - -export const test = base.extend({ - loginPage: async ({ page }, use) => { - await use(new LoginPage(page)); - }, - dashboardLinks: async ({ page }, use) => { - await use(new DashboardLinks(page)); - }, - virtualKeysPage: async ({ page }, use) => { - await use(new VirtualKeysPage(page)); - }, - modelsPage: async ({ page }, use) => { - await use(new ModelsPage(page)); - }, -}); -export { expect } from "@playwright/test"; diff --git a/ui/litellm-dashboard/tests/package-lock.json b/ui/litellm-dashboard/tests/package-lock.json deleted file mode 100644 index 19ade17417..0000000000 --- a/ui/litellm-dashboard/tests/package-lock.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "name": "tests", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "tests", - "version": "1.0.0", - "license": "ISC", - "dependencies": { - "dotenv": "^16.4.7" - }, - "devDependencies": { - "@playwright/test": "^1.51.0", - "@types/node": "^22.13.10" - } - }, - "node_modules/@playwright/test": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.51.0.tgz", - "integrity": "sha512-dJ0dMbZeHhI+wb77+ljx/FeC8VBP6j/rj9OAojO08JI80wTZy6vRk9KvHKiDCUh4iMpEiseMgqRBIeW+eKX6RA==", - "dev": true, - "dependencies": { - "playwright": "1.51.0" - }, - "bin": { - "playwright": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@types/node": { - "version": "22.13.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz", - "integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==", - "dev": true, - "dependencies": { - "undici-types": "~6.20.0" - } - }, - "node_modules/dotenv": { - "version": "16.4.7", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", - "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/playwright": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.51.0.tgz", - "integrity": "sha512-442pTfGM0xxfCYxuBa/Pu6B2OqxqqaYq39JS8QDMGThUvIOCd6s0ANDog3uwA0cHavVlnTQzGCN7Id2YekDSXA==", - "dev": true, - "dependencies": { - "playwright-core": "1.51.0" - }, - "bin": { - "playwright": "cli.js" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" - } - }, - "node_modules/playwright-core": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.51.0.tgz", - "integrity": "sha512-x47yPE3Zwhlil7wlNU/iktF7t2r/URR3VLbH6EknJd/04Qc/PSJ0EY3CMXipmglLG+zyRxW6HNo2EGbKLHPWMg==", - "dev": true, - "bin": { - "playwright-core": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "dev": true - } - } -} diff --git a/ui/litellm-dashboard/tests/package.json b/ui/litellm-dashboard/tests/package.json deleted file mode 100644 index 650bf45548..0000000000 --- a/ui/litellm-dashboard/tests/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "tests", - "version": "1.0.0", - "main": "index.js", - "scripts": {}, - "keywords": [], - "author": "", - "license": "ISC", - "description": "", - "devDependencies": { - "@playwright/test": "^1.51.0", - "@types/node": "^22.13.10" - }, - "dependencies": { - "dotenv": "^16.4.7" - } -} diff --git a/ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts b/ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts deleted file mode 100644 index c9b14852ea..0000000000 --- a/ui/litellm-dashboard/tests/page-object-models/dashboard-links.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Page, Locator } from "@playwright/test"; - -export class DashboardLinks { - private readonly userButton: Locator; - private readonly logoutButton: Locator; - private readonly modelsPageLink: Locator; - - constructor(private readonly page: Page) { - this.userButton = this.page.getByRole("button", { name: "User" }); - this.logoutButton = this.page.getByText("Logout"); - this.modelsPageLink = this.page.getByRole("menuitem", { - name: "block Models", - }); - } - - async logout() { - await this.userButton.click(); - await this.logoutButton.click(); - } - - getUserButton(): Locator { - return this.userButton; - } - - getModelsPageLink(): Locator { - return this.modelsPageLink; - } -} diff --git a/ui/litellm-dashboard/tests/page-object-models/login.page.ts b/ui/litellm-dashboard/tests/page-object-models/login.page.ts deleted file mode 100644 index 14df36b5e4..0000000000 --- a/ui/litellm-dashboard/tests/page-object-models/login.page.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { Page, Locator } from "@playwright/test"; - -export class LoginPage { - //Locators as fields - private readonly usernameInput: Locator; - private readonly passwordInput: Locator; - private readonly loginSubmit: Locator; - - //Initialize locators in constructor - constructor(private readonly page: Page) { - this.usernameInput = this.page.getByRole("textbox", { name: "Username:" }); - this.passwordInput = this.page.getByRole("textbox", { name: "Password:" }); - this.loginSubmit = this.page.getByRole("button", { name: "Submit" }); - } - - async goto() { - await this.page.goto("/ui"); - } - - async login(username: string, password: string) { - await this.usernameInput.click(); - await this.usernameInput.fill(username); - await this.passwordInput.click(); - await this.passwordInput.fill(password); - await this.loginSubmit.click(); - } -} diff --git a/ui/litellm-dashboard/tests/page-object-models/models.page.ts b/ui/litellm-dashboard/tests/page-object-models/models.page.ts deleted file mode 100644 index f0c0ae3c13..0000000000 --- a/ui/litellm-dashboard/tests/page-object-models/models.page.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { Page, Locator } from "@playwright/test"; - -export class ModelsPage { - // Models Page Tabs - private readonly allModelsTab: Locator; - private readonly addModelTab: Locator; - // All Models Tab Locators - // Add Model Tab Form Locators - private readonly providerCombobox: Locator; - // *private openaiProviderComboboxOption: Locator; - private readonly litellmModelNameCombobox: Locator; - // *private readonly litellmModelNameComboboxOption page.getByTitle('omni-moderation-latest', { exact: true }).locator('div') - /*private readonly modelMappingPublicNameInput: Locator;*/ - private readonly apiKeyInput: Locator; - private readonly addModelSubmitButton: Locator; - - constructor(private readonly page: Page) { - // Models Page Tabs - this.allModelsTab = this.page.getByRole("tab", { name: "All Models" }); - this.addModelTab = this.page.getByRole("tab", { name: "Add Model" }); - // Add Model Tab Form Locators - this.providerCombobox = this.page.getByRole("combobox", { - name: "* Provider question-circle :", - }); - /**this.openaiProviderComboboxOption = this.page - .locator("span") - .filter({ hasText: "OpenAI" });*/ - this.litellmModelNameCombobox = this.page.locator("#model"); - /*this.modelMappingPublicNameInput = this.page - .getByRole("row", { name: "omni-moderation-latest omni-" }) - .getByTestId("base-input");*/ - this.apiKeyInput = page.getByRole("textbox", { - name: "* API Key question-circle :", - }); - this.addModelSubmitButton = page.getByRole("button", { name: "Add Model" }); - } - - // 'All Model' Tab // - getAllModelsTab(): Locator { - return this.allModelsTab; - } - - // Parametized Locators - getAllModelsTableCellValue(allModelsTableCellValue: string): Locator { - return this.page - .getByRole("cell", { name: allModelsTableCellValue }) - .first(); - } - - // 'Add Model' Tab // - getAddModelTab(): Locator { - return this.addModelTab; - } - - // Parametized Form Locators - /*getProviderComboboxOption(providerComboboxOption: string): Locator { - this.page - .locator("span") - .filter({ hasText: providerComboboxOption }); - }*/ - - fillProviderComboboxBox(providerComboboxText: string) { - this.page - .getByRole("combobox", { name: "* Provider question-circle :" }) - .fill(providerComboboxText); - } - - getLitellmModelNameCombobox(): Locator { - return this.litellmModelNameCombobox; - } - - /*getLitellmModelNameComboboxOption( - litellmModelNameComboboxOption: string - ): Locator { - return this.page - .getByTitle(litellmModelNameComboboxOption, { exact: true }) - .locator("div"); - }*/ - - fillLitellmModelNameCombobox(litellmModelNameComboboxOption: string) { - this.page.locator("#model").fill(litellmModelNameComboboxOption); - } - - getLitellmModelMappingModel(litellmModelMappingModel: string): Locator { - return this.page - .locator("#model_mappings") - .getByText(litellmModelMappingModel); - } - - getLitellmModelMappingModelPublicName( - litellmModelMappingModel: string - ): Locator { - return this.page - .getByRole("row", { name: litellmModelMappingModel }) - .getByTestId("base-input"); - } - - // Non-parametized Form Locators - getProviderCombobox(): Locator { - return this.providerCombobox; - } - - getLitellModelNameCombobox(): Locator { - return this.litellmModelNameCombobox; - } - - getAPIKeyInputBox(provider: string): Locator { - return this.page.getByRole("textbox", { name: `* ${provider} API Key :` }); - } - - getAddModelSubmitButton(): Locator { - return this.addModelSubmitButton; - } -} diff --git a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts b/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts deleted file mode 100644 index 0f7949f2ed..0000000000 --- a/ui/litellm-dashboard/tests/page-object-models/virtual-keys.page.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { Page, Locator } from "@playwright/test"; - -export class VirtualKeysPage { - private readonly userButton: Locator; - private readonly logoutButton: Locator; - private readonly createNewKeyButton: Locator; - private readonly ownedByYouRadioButton: Locator; - private readonly keyNameInput: Locator; - private readonly modelInput: Locator; - private readonly createKeyButton: Locator; - private readonly copyAPIKeyButton: Locator; - - constructor(private readonly page: Page) { - this.userButton = this.page.getByRole("button", { name: "User" }); - this.logoutButton = this.page.getByText("Logout"); - this.createNewKeyButton = this.page.getByRole("button", { - name: "+ Create New Key", - }); - this.ownedByYouRadioButton = this.page.getByRole("radio", { name: "You" }); - this.keyNameInput = this.page.getByTestId("base-input"); - this.modelInput = this.page.locator(".ant-select-selection-overflow"); - this.createKeyButton = this.page.getByRole("button", { - name: "Create Key", - }); - this.copyAPIKeyButton = this.page.getByRole("button", { - name: "Copy API Key", - }); - } - - async logout() { - await this.userButton.click(); - await this.logoutButton.click(); - } - - getUserButton(): Locator { - return this.userButton; - } - - getCreateNewKeyButton(): Locator { - return this.createNewKeyButton; - } - - getVirtualKeysTableCellValue(virtualKeysTableCellValue: string): Locator { - return this.page.getByRole("cell", { name: virtualKeysTableCellValue }); - } - - getOwnedByYouRadioButton(): Locator { - return this.ownedByYouRadioButton; - } - - getKeyNameInput(): Locator { - return this.keyNameInput; - } - - getModelInput(): Locator { - return this.modelInput; - } - - getCreateKeyButton(): Locator { - return this.createKeyButton; - } - - getCopyAPIKeyButton(): Locator { - return this.copyAPIKeyButton; - } -} diff --git a/ui/litellm-dashboard/tests/playwright.config.ts b/ui/litellm-dashboard/tests/playwright.config.ts deleted file mode 100644 index ef0f64237d..0000000000 --- a/ui/litellm-dashboard/tests/playwright.config.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { defineConfig, devices } from "@playwright/test"; -/** - * See https://playwright.dev/docs/test-configuration. - */ - -// import { config } from "dotenv"; -// // import path from "path"; -// config({ path: "./../../../.env.example" }); - -export default defineConfig({ - globalSetup: "utils/globalSetup.ts", - testDir: "./e2e", - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - // workers: process.env.CI ? 1 : undefined, - workers: 1, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: "html", - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: "http://localhost:4000", - - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: "on-first-retry", - - contextOptions: { - permissions: ["clipboard-read"], - }, - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: "chromium", - use: { ...devices["Desktop Chrome"] }, - }, - /* - { - name: "firefox", - use: { ...devices["Desktop Firefox"] }, - }, - - { - name: "webkit", - use: { ...devices["Desktop Safari"] }, - },*/ - - /* Test against mobile viewports. */ - // { - // name: 'Mobile Chrome', - // use: { ...devices['Pixel 5'] }, - // }, - // { - // name: 'Mobile Safari', - // use: { ...devices['iPhone 12'] }, - // }, - - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { ...devices['Desktop Edge'], channel: 'msedge' }, - // }, - // { - // name: 'Google Chrome', - // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - // }, - ], - - /* Run your local dev server before starting the tests */ - // webServer: { - // command: 'npm run start', - // url: 'http://127.0.0.1:3000', - // reuseExistingServer: !process.env.CI, - // }, -}); diff --git a/ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts b/ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts deleted file mode 100644 index 8641cb5f5d..0000000000 --- a/ui/litellm-dashboard/tests/tests-examples/demo-todo-app.spec.ts +++ /dev/null @@ -1,437 +0,0 @@ -import { test, expect, type Page } from '@playwright/test'; - -test.beforeEach(async ({ page }) => { - await page.goto('https://demo.playwright.dev/todomvc'); -}); - -const TODO_ITEMS = [ - 'buy some cheese', - 'feed the cat', - 'book a doctors appointment' -] as const; - -test.describe('New Todo', () => { - test('should allow me to add todo items', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create 1st todo. - await newTodo.fill(TODO_ITEMS[0]); - await newTodo.press('Enter'); - - // Make sure the list only has one todo item. - await expect(page.getByTestId('todo-title')).toHaveText([ - TODO_ITEMS[0] - ]); - - // Create 2nd todo. - await newTodo.fill(TODO_ITEMS[1]); - await newTodo.press('Enter'); - - // Make sure the list now has two todo items. - await expect(page.getByTestId('todo-title')).toHaveText([ - TODO_ITEMS[0], - TODO_ITEMS[1] - ]); - - await checkNumberOfTodosInLocalStorage(page, 2); - }); - - test('should clear text input field when an item is added', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create one todo item. - await newTodo.fill(TODO_ITEMS[0]); - await newTodo.press('Enter'); - - // Check that input is empty. - await expect(newTodo).toBeEmpty(); - await checkNumberOfTodosInLocalStorage(page, 1); - }); - - test('should append new items to the bottom of the list', async ({ page }) => { - // Create 3 items. - await createDefaultTodos(page); - - // create a todo count locator - const todoCount = page.getByTestId('todo-count') - - // Check test using different methods. - await expect(page.getByText('3 items left')).toBeVisible(); - await expect(todoCount).toHaveText('3 items left'); - await expect(todoCount).toContainText('3'); - await expect(todoCount).toHaveText(/3/); - - // Check all items in one call. - await expect(page.getByTestId('todo-title')).toHaveText(TODO_ITEMS); - await checkNumberOfTodosInLocalStorage(page, 3); - }); -}); - -test.describe('Mark all as completed', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test.afterEach(async ({ page }) => { - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test('should allow me to mark all items as completed', async ({ page }) => { - // Complete all todos. - await page.getByLabel('Mark all as complete').check(); - - // Ensure all todos have 'completed' class. - await expect(page.getByTestId('todo-item')).toHaveClass(['completed', 'completed', 'completed']); - await checkNumberOfCompletedTodosInLocalStorage(page, 3); - }); - - test('should allow me to clear the complete state of all items', async ({ page }) => { - const toggleAll = page.getByLabel('Mark all as complete'); - // Check and then immediately uncheck. - await toggleAll.check(); - await toggleAll.uncheck(); - - // Should be no completed classes. - await expect(page.getByTestId('todo-item')).toHaveClass(['', '', '']); - }); - - test('complete all checkbox should update state when items are completed / cleared', async ({ page }) => { - const toggleAll = page.getByLabel('Mark all as complete'); - await toggleAll.check(); - await expect(toggleAll).toBeChecked(); - await checkNumberOfCompletedTodosInLocalStorage(page, 3); - - // Uncheck first todo. - const firstTodo = page.getByTestId('todo-item').nth(0); - await firstTodo.getByRole('checkbox').uncheck(); - - // Reuse toggleAll locator and make sure its not checked. - await expect(toggleAll).not.toBeChecked(); - - await firstTodo.getByRole('checkbox').check(); - await checkNumberOfCompletedTodosInLocalStorage(page, 3); - - // Assert the toggle all is checked again. - await expect(toggleAll).toBeChecked(); - }); -}); - -test.describe('Item', () => { - - test('should allow me to mark items as complete', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create two items. - for (const item of TODO_ITEMS.slice(0, 2)) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } - - // Check first item. - const firstTodo = page.getByTestId('todo-item').nth(0); - await firstTodo.getByRole('checkbox').check(); - await expect(firstTodo).toHaveClass('completed'); - - // Check second item. - const secondTodo = page.getByTestId('todo-item').nth(1); - await expect(secondTodo).not.toHaveClass('completed'); - await secondTodo.getByRole('checkbox').check(); - - // Assert completed class. - await expect(firstTodo).toHaveClass('completed'); - await expect(secondTodo).toHaveClass('completed'); - }); - - test('should allow me to un-mark items as complete', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // Create two items. - for (const item of TODO_ITEMS.slice(0, 2)) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } - - const firstTodo = page.getByTestId('todo-item').nth(0); - const secondTodo = page.getByTestId('todo-item').nth(1); - const firstTodoCheckbox = firstTodo.getByRole('checkbox'); - - await firstTodoCheckbox.check(); - await expect(firstTodo).toHaveClass('completed'); - await expect(secondTodo).not.toHaveClass('completed'); - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - - await firstTodoCheckbox.uncheck(); - await expect(firstTodo).not.toHaveClass('completed'); - await expect(secondTodo).not.toHaveClass('completed'); - await checkNumberOfCompletedTodosInLocalStorage(page, 0); - }); - - test('should allow me to edit an item', async ({ page }) => { - await createDefaultTodos(page); - - const todoItems = page.getByTestId('todo-item'); - const secondTodo = todoItems.nth(1); - await secondTodo.dblclick(); - await expect(secondTodo.getByRole('textbox', { name: 'Edit' })).toHaveValue(TODO_ITEMS[1]); - await secondTodo.getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); - await secondTodo.getByRole('textbox', { name: 'Edit' }).press('Enter'); - - // Explicitly assert the new text value. - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - 'buy some sausages', - TODO_ITEMS[2] - ]); - await checkTodosInLocalStorage(page, 'buy some sausages'); - }); -}); - -test.describe('Editing', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test('should hide other controls when editing', async ({ page }) => { - const todoItem = page.getByTestId('todo-item').nth(1); - await todoItem.dblclick(); - await expect(todoItem.getByRole('checkbox')).not.toBeVisible(); - await expect(todoItem.locator('label', { - hasText: TODO_ITEMS[1], - })).not.toBeVisible(); - await checkNumberOfTodosInLocalStorage(page, 3); - }); - - test('should save edits on blur', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).dispatchEvent('blur'); - - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - 'buy some sausages', - TODO_ITEMS[2], - ]); - await checkTodosInLocalStorage(page, 'buy some sausages'); - }); - - test('should trim entered text', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill(' buy some sausages '); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Enter'); - - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - 'buy some sausages', - TODO_ITEMS[2], - ]); - await checkTodosInLocalStorage(page, 'buy some sausages'); - }); - - test('should remove the item if an empty text string was entered', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill(''); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Enter'); - - await expect(todoItems).toHaveText([ - TODO_ITEMS[0], - TODO_ITEMS[2], - ]); - }); - - test('should cancel edits on escape', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).dblclick(); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).fill('buy some sausages'); - await todoItems.nth(1).getByRole('textbox', { name: 'Edit' }).press('Escape'); - await expect(todoItems).toHaveText(TODO_ITEMS); - }); -}); - -test.describe('Counter', () => { - test('should display the current number of todo items', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - // create a todo count locator - const todoCount = page.getByTestId('todo-count') - - await newTodo.fill(TODO_ITEMS[0]); - await newTodo.press('Enter'); - - await expect(todoCount).toContainText('1'); - - await newTodo.fill(TODO_ITEMS[1]); - await newTodo.press('Enter'); - await expect(todoCount).toContainText('2'); - - await checkNumberOfTodosInLocalStorage(page, 2); - }); -}); - -test.describe('Clear completed button', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - }); - - test('should display the correct text', async ({ page }) => { - await page.locator('.todo-list li .toggle').first().check(); - await expect(page.getByRole('button', { name: 'Clear completed' })).toBeVisible(); - }); - - test('should remove completed items when clicked', async ({ page }) => { - const todoItems = page.getByTestId('todo-item'); - await todoItems.nth(1).getByRole('checkbox').check(); - await page.getByRole('button', { name: 'Clear completed' }).click(); - await expect(todoItems).toHaveCount(2); - await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[2]]); - }); - - test('should be hidden when there are no items that are completed', async ({ page }) => { - await page.locator('.todo-list li .toggle').first().check(); - await page.getByRole('button', { name: 'Clear completed' }).click(); - await expect(page.getByRole('button', { name: 'Clear completed' })).toBeHidden(); - }); -}); - -test.describe('Persistence', () => { - test('should persist its data', async ({ page }) => { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - for (const item of TODO_ITEMS.slice(0, 2)) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } - - const todoItems = page.getByTestId('todo-item'); - const firstTodoCheck = todoItems.nth(0).getByRole('checkbox'); - await firstTodoCheck.check(); - await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[1]]); - await expect(firstTodoCheck).toBeChecked(); - await expect(todoItems).toHaveClass(['completed', '']); - - // Ensure there is 1 completed item. - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - - // Now reload. - await page.reload(); - await expect(todoItems).toHaveText([TODO_ITEMS[0], TODO_ITEMS[1]]); - await expect(firstTodoCheck).toBeChecked(); - await expect(todoItems).toHaveClass(['completed', '']); - }); -}); - -test.describe('Routing', () => { - test.beforeEach(async ({ page }) => { - await createDefaultTodos(page); - // make sure the app had a chance to save updated todos in storage - // before navigating to a new view, otherwise the items can get lost :( - // in some frameworks like Durandal - await checkTodosInLocalStorage(page, TODO_ITEMS[0]); - }); - - test('should allow me to display active items', async ({ page }) => { - const todoItem = page.getByTestId('todo-item'); - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - await page.getByRole('link', { name: 'Active' }).click(); - await expect(todoItem).toHaveCount(2); - await expect(todoItem).toHaveText([TODO_ITEMS[0], TODO_ITEMS[2]]); - }); - - test('should respect the back button', async ({ page }) => { - const todoItem = page.getByTestId('todo-item'); - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - - await test.step('Showing all items', async () => { - await page.getByRole('link', { name: 'All' }).click(); - await expect(todoItem).toHaveCount(3); - }); - - await test.step('Showing active items', async () => { - await page.getByRole('link', { name: 'Active' }).click(); - }); - - await test.step('Showing completed items', async () => { - await page.getByRole('link', { name: 'Completed' }).click(); - }); - - await expect(todoItem).toHaveCount(1); - await page.goBack(); - await expect(todoItem).toHaveCount(2); - await page.goBack(); - await expect(todoItem).toHaveCount(3); - }); - - test('should allow me to display completed items', async ({ page }) => { - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - await page.getByRole('link', { name: 'Completed' }).click(); - await expect(page.getByTestId('todo-item')).toHaveCount(1); - }); - - test('should allow me to display all items', async ({ page }) => { - await page.getByTestId('todo-item').nth(1).getByRole('checkbox').check(); - await checkNumberOfCompletedTodosInLocalStorage(page, 1); - await page.getByRole('link', { name: 'Active' }).click(); - await page.getByRole('link', { name: 'Completed' }).click(); - await page.getByRole('link', { name: 'All' }).click(); - await expect(page.getByTestId('todo-item')).toHaveCount(3); - }); - - test('should highlight the currently applied filter', async ({ page }) => { - await expect(page.getByRole('link', { name: 'All' })).toHaveClass('selected'); - - //create locators for active and completed links - const activeLink = page.getByRole('link', { name: 'Active' }); - const completedLink = page.getByRole('link', { name: 'Completed' }); - await activeLink.click(); - - // Page change - active items. - await expect(activeLink).toHaveClass('selected'); - await completedLink.click(); - - // Page change - completed items. - await expect(completedLink).toHaveClass('selected'); - }); -}); - -async function createDefaultTodos(page: Page) { - // create a new todo locator - const newTodo = page.getByPlaceholder('What needs to be done?'); - - for (const item of TODO_ITEMS) { - await newTodo.fill(item); - await newTodo.press('Enter'); - } -} - -async function checkNumberOfTodosInLocalStorage(page: Page, expected: number) { - return await page.waitForFunction(e => { - return JSON.parse(localStorage['react-todos']).length === e; - }, expected); -} - -async function checkNumberOfCompletedTodosInLocalStorage(page: Page, expected: number) { - return await page.waitForFunction(e => { - return JSON.parse(localStorage['react-todos']).filter((todo: any) => todo.completed).length === e; - }, expected); -} - -async function checkTodosInLocalStorage(page: Page, title: string) { - return await page.waitForFunction(t => { - return JSON.parse(localStorage['react-todos']).map((todo: any) => todo.title).includes(t); - }, title); -} diff --git a/ui/litellm-dashboard/tests/utils/globalSetup.ts b/ui/litellm-dashboard/tests/utils/globalSetup.ts deleted file mode 100644 index 2fb0a8f956..0000000000 --- a/ui/litellm-dashboard/tests/utils/globalSetup.ts +++ /dev/null @@ -1,8 +0,0 @@ -import dotenv from "dotenv"; - -export default async function globalSetup() { - dotenv.config({ - //path should be relative to playwright.config.ts - path: "./../../../.env", - }); -} diff --git a/ui/litellm-dashboard/tests/utils/utils.ts b/ui/litellm-dashboard/tests/utils/utils.ts deleted file mode 100644 index 8261063859..0000000000 --- a/ui/litellm-dashboard/tests/utils/utils.ts +++ /dev/null @@ -1,14 +0,0 @@ -// import * as dotenv from 'dotenv'; -// import { config } from "dotenv"; -// import path from "path"; -// config({ path: "./../../../../.env.example" }); - -export function loginDetailsSet(): Boolean { - // console.log(process.env.DATABASE_URL); - // console.log(process.env.UI_PASSWORD); - let loginDetailsSet = false; - if (process.env.UI_USERNAME && process.env.UI_PASSWORD) { - loginDetailsSet = true; - } - return loginDetailsSet; -} From 179ecc8fae442718e1b6614b5d425cde8268dcfb Mon Sep 17 00:00:00 2001 From: omnisilica Date: Mon, 31 Mar 2025 23:18:03 -0400 Subject: [PATCH 10/14] Transferred e2e tests and cleaned reworked existing e2e files --- .../data/providers-and-models.json | 26 +++ .../e2e_ui_tests/add-model.spec.ts | 162 ++++++++++++++++++ .../e2e_ui_tests/create-api-key.spec.ts | 76 ++++++++ .../e2e_ui_tests/login.spec.ts | 29 ++++ .../proxy_admin_ui_tests/fixtures/fixtures.ts | 28 +++ .../page-object-models/dashboard-links.ts | 28 +++ .../page-object-models/login.page.ts | 27 +++ .../page-object-models/models.page.ts | 114 ++++++++++++ .../page-object-models/virtual-keys.page.ts | 66 +++++++ .../proxy_admin_ui_tests/utils/globalSetup.ts | 8 + tests/proxy_admin_ui_tests/utils/utils.ts | 14 ++ 11 files changed, 578 insertions(+) create mode 100644 tests/proxy_admin_ui_tests/data/providers-and-models.json create mode 100644 tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts create mode 100644 tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts create mode 100644 tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts create mode 100644 tests/proxy_admin_ui_tests/fixtures/fixtures.ts create mode 100644 tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts create mode 100644 tests/proxy_admin_ui_tests/page-object-models/login.page.ts create mode 100644 tests/proxy_admin_ui_tests/page-object-models/models.page.ts create mode 100644 tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts create mode 100644 tests/proxy_admin_ui_tests/utils/globalSetup.ts create mode 100644 tests/proxy_admin_ui_tests/utils/utils.ts diff --git a/tests/proxy_admin_ui_tests/data/providers-and-models.json b/tests/proxy_admin_ui_tests/data/providers-and-models.json new file mode 100644 index 0000000000..95a1e37e8f --- /dev/null +++ b/tests/proxy_admin_ui_tests/data/providers-and-models.json @@ -0,0 +1,26 @@ +{ + "OpenAI": [ + "Custom Model Name (Enter below)", + "All OpenAI Models (Wildcard)", + "omni-moderation-latest", + "omni-moderation-latest-intents", + "omni-moderation-2024-09-26", + "gpt-4", + "gpt-4o", + "gpt-4o-search-preview-2025-03-11", + "gpt-4o-search-preview", + "gpt-4.5-preview" + ], + "Anthropic": [ + "Custom Model Name (Enter below)", + "All Anthropic Models (Wildcard)", + "claude-instant-1", + "claude-instant-1.2", + "claude-2", + "claude-2.1", + "claude-3-haiku-20240307", + "claude-3-5-haiku-20241022", + "claude-3-5-haiku-latest", + "claude-3-opus-latest" + ] +} diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts new file mode 100644 index 0000000000..02d2425cf4 --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts @@ -0,0 +1,162 @@ +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; +const providersAndModels = JSON.parse( + JSON.stringify(require("./../data/providers-and-models.json")) +); + +providersAndModels["OpenAI"].forEach((model: string) => { + test(`4644_Test_Adding_OpenAI's_${model}_model`, async ({ + loginPage, + dashboardLinks, + modelsPage, + page, + }) => { + const excludeLitellmModelNameDropdownValues = [ + "Custom Model Name (Enter below)", + "All OpenAI Models (Wildcard)", + ]; + if (!excludeLitellmModelNameDropdownValues.includes(model)) { + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); + + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); + + // console.log("4. Selecting OpenAI from 'Provider' dropdown"); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox("OpenAI"); + await modelsPage.getProviderCombobox().press("Enter"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); + + // console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); + await modelsPage.getLitellModelNameCombobox().click(); + await modelsPage.getLitellModelNameCombobox().fill(model); + await modelsPage.getLitellModelNameCombobox().press("Enter"); + await expect(modelsPage.getLitellmModelMappingModel(model)).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/05_select-${model}-model.png`,}); + + // console.log("6. Adding API Key"); + await modelsPage.getAPIKeyInputBox("OpenAI").click(); + await modelsPage.getAPIKeyInputBox("OpenAI").fill("sk-1234"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/06_enter-api-key.png`,}); + + // console.log("7. Clicking 'Add Model'"); + await modelsPage.getAddModelSubmitButton().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/07_add-model.png`,}); + + // console.log("8. Navigating to 'All Models'"); + if (model.length > 20) { + model = model.slice(0, 20) + "..."; + } + await modelsPage.getAllModelsTab().click(); + + await expect( + modelsPage.getAllModelsTableCellValue(`openai logo openai`) + ).toBeVisible(); + await expect(modelsPage.getAllModelsTableCellValue(model)).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/08_navigate-to-all-models-tab.png`,}); + + // console.log("Clean Up - Delete Model Created"); + const modelID = await page + .locator("tr.tremor-TableRow-row.h-8") + .nth(1) + .locator(".tremor-Button-text.text-tremor-default") + .innerText(); + + await page.getByRole("button", { name: modelID }).click(); + await page.getByRole("button", { name: "Delete Model" }).click(); + await page.getByRole("button", { name: "Delete", exact: true }).click(); + + // console.log("9. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`,}); + } + }); +}); + +Object.entries(providersAndModels).forEach(([provider, model]) => { + test(`4644_Test_the_Correct_Dropdown_Shows_When_Adding_${provider}_Models`, async ({ + loginPage, + dashboardLinks, + modelsPage, + page, + }) => { + let username = "admin"; + let password = "sk-1234"; + const excludeLitellmModelNameDropdownValues = [ + "OpenAI", + "OpenAI-Compatible Endpoints (Together AI, etc.)", + "OpenAI Text Completion", + "OpenAI-Compatible Text Completion Models (Together AI, etc.)", + "Anthropic", + ]; + let litellmModelNameDropdownValues: string[] = []; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); + + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); + + // console.log(`4. Selecting ${model} from 'Provider' dropdown`); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox(provider); + await modelsPage.getProviderCombobox().press("Enter"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); + + //Scrape ant-selection-option and add to list + await modelsPage.getLitellModelNameCombobox().click(); + const litellmModelOptions = await page + .locator(".rc-virtual-list-holder-inner") + .locator(".ant-select-item-option-content") + .all(); + + for (const element of litellmModelOptions) { + let modelNameDropdownValue = await element.innerText(); + if ( + !excludeLitellmModelNameDropdownValues.includes(modelNameDropdownValue) + ) { + litellmModelNameDropdownValues.push(modelNameDropdownValue); + } + } + litellmModelNameDropdownValues.forEach((element) => { + expect(providersAndModels[provider].includes(element)).toBeTruthy(); + }); + + // console.log("5. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); + }); +}); diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts new file mode 100644 index 0000000000..9e67f06863 --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts @@ -0,0 +1,76 @@ +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ + loginPage, + virtualKeysPage, + page, +}) => { + let username = "admin"; + let password = "sk-1234"; + // let apiKey = ""; + let apiKeyID = ""; + const keyName = "test-key-name-3"; + + if (loginDetailsSet()) { + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Clicking the '+ Create New Key' button"); + await virtualKeysPage.getCreateNewKeyButton().click(); + + // console.log("3. Clicking the Owned By You radio button."); + await virtualKeysPage.getOwnedByYouRadioButton().check(); + + // console.log("4. Entering a key name in the 'Key Name' input."); + await virtualKeysPage.getKeyNameInput().fill(keyName); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/04_enter-key-name.png`,}); + + // console.log("5. Selecting All Team Models"); + await virtualKeysPage.getModelInput().click(); + await page.getByText("All Team Models").click(); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/05_select-all-team-models.png`,}); + + // console.log("6. Clicking 'Create Key'"); + await virtualKeysPage.getCreateKeyButton().click(); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/06_create-api-key.png`,}); + + // console.log("7. Copying the API key to clipboard"); + /* + await virtualKeysPage.getCopyAPIKeyButton().click(); + apiKey = await page.evaluate(async () => { + return await navigator.clipboard.readText(); + }); + */ + + // console.log("8. Exiting Modal Window"); + await page + .getByRole("dialog") + .filter({ hasText: "Save your KeyPlease save this" }) + .getByLabel("Close", { exact: true }) + .click(); + await expect( + virtualKeysPage.getVirtualKeysTableCellValue(keyName) + ).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/08_check-api-key-created.png`,}); + + apiKeyID = await page + .locator("tr.tremor-TableRow-row.h-8") + .locator(".tremor-Button-text.text-tremor-default") + .innerText(); + await page.getByRole("button", { name: apiKeyID }).click(); + await page.getByRole("button", { name: "Delete Key" }).click(); + await page.getByRole("button", { name: "Delete", exact: true }).click(); + + // console.log("9. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); +}); diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts new file mode 100644 index 0000000000..a7de95d575 --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts @@ -0,0 +1,29 @@ +import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +test("4644_Test_Basic_Sign_in_Flow", async ({ + loginPage, + virtualKeysPage, + page, +}) => { + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + await loginPage.goto(); + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/navigate-to-login-page.png",}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/dashboard.png",}); + + await virtualKeysPage.logout(); + await expect( + page.getByRole("heading", { name: "LiteLLM Login" }) + ).toBeVisible(); + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/logout.png",}); +}); diff --git a/tests/proxy_admin_ui_tests/fixtures/fixtures.ts b/tests/proxy_admin_ui_tests/fixtures/fixtures.ts new file mode 100644 index 0000000000..b7342d1153 --- /dev/null +++ b/tests/proxy_admin_ui_tests/fixtures/fixtures.ts @@ -0,0 +1,28 @@ +import { DashboardLinks } from "./../page-object-models/dashboard-links"; +import { VirtualKeysPage } from "./../page-object-models/virtual-keys.page"; +import { test as base } from "@playwright/test"; +import { LoginPage } from "../page-object-models/login.page"; +import { ModelsPage } from "../page-object-models/models.page"; + +type Fixtures = { + loginPage: LoginPage; + dashboardLinks: DashboardLinks; + virtualKeysPage: VirtualKeysPage; + modelsPage: ModelsPage; +}; + +export const test = base.extend({ + loginPage: async ({ page }, use) => { + await use(new LoginPage(page)); + }, + dashboardLinks: async ({ page }, use) => { + await use(new DashboardLinks(page)); + }, + virtualKeysPage: async ({ page }, use) => { + await use(new VirtualKeysPage(page)); + }, + modelsPage: async ({ page }, use) => { + await use(new ModelsPage(page)); + }, +}); +export { expect } from "@playwright/test"; diff --git a/tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts b/tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts new file mode 100644 index 0000000000..c9b14852ea --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts @@ -0,0 +1,28 @@ +import { Page, Locator } from "@playwright/test"; + +export class DashboardLinks { + private readonly userButton: Locator; + private readonly logoutButton: Locator; + private readonly modelsPageLink: Locator; + + constructor(private readonly page: Page) { + this.userButton = this.page.getByRole("button", { name: "User" }); + this.logoutButton = this.page.getByText("Logout"); + this.modelsPageLink = this.page.getByRole("menuitem", { + name: "block Models", + }); + } + + async logout() { + await this.userButton.click(); + await this.logoutButton.click(); + } + + getUserButton(): Locator { + return this.userButton; + } + + getModelsPageLink(): Locator { + return this.modelsPageLink; + } +} diff --git a/tests/proxy_admin_ui_tests/page-object-models/login.page.ts b/tests/proxy_admin_ui_tests/page-object-models/login.page.ts new file mode 100644 index 0000000000..14df36b5e4 --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/login.page.ts @@ -0,0 +1,27 @@ +import type { Page, Locator } from "@playwright/test"; + +export class LoginPage { + //Locators as fields + private readonly usernameInput: Locator; + private readonly passwordInput: Locator; + private readonly loginSubmit: Locator; + + //Initialize locators in constructor + constructor(private readonly page: Page) { + this.usernameInput = this.page.getByRole("textbox", { name: "Username:" }); + this.passwordInput = this.page.getByRole("textbox", { name: "Password:" }); + this.loginSubmit = this.page.getByRole("button", { name: "Submit" }); + } + + async goto() { + await this.page.goto("/ui"); + } + + async login(username: string, password: string) { + await this.usernameInput.click(); + await this.usernameInput.fill(username); + await this.passwordInput.click(); + await this.passwordInput.fill(password); + await this.loginSubmit.click(); + } +} diff --git a/tests/proxy_admin_ui_tests/page-object-models/models.page.ts b/tests/proxy_admin_ui_tests/page-object-models/models.page.ts new file mode 100644 index 0000000000..f0c0ae3c13 --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/models.page.ts @@ -0,0 +1,114 @@ +import { Page, Locator } from "@playwright/test"; + +export class ModelsPage { + // Models Page Tabs + private readonly allModelsTab: Locator; + private readonly addModelTab: Locator; + // All Models Tab Locators + // Add Model Tab Form Locators + private readonly providerCombobox: Locator; + // *private openaiProviderComboboxOption: Locator; + private readonly litellmModelNameCombobox: Locator; + // *private readonly litellmModelNameComboboxOption page.getByTitle('omni-moderation-latest', { exact: true }).locator('div') + /*private readonly modelMappingPublicNameInput: Locator;*/ + private readonly apiKeyInput: Locator; + private readonly addModelSubmitButton: Locator; + + constructor(private readonly page: Page) { + // Models Page Tabs + this.allModelsTab = this.page.getByRole("tab", { name: "All Models" }); + this.addModelTab = this.page.getByRole("tab", { name: "Add Model" }); + // Add Model Tab Form Locators + this.providerCombobox = this.page.getByRole("combobox", { + name: "* Provider question-circle :", + }); + /**this.openaiProviderComboboxOption = this.page + .locator("span") + .filter({ hasText: "OpenAI" });*/ + this.litellmModelNameCombobox = this.page.locator("#model"); + /*this.modelMappingPublicNameInput = this.page + .getByRole("row", { name: "omni-moderation-latest omni-" }) + .getByTestId("base-input");*/ + this.apiKeyInput = page.getByRole("textbox", { + name: "* API Key question-circle :", + }); + this.addModelSubmitButton = page.getByRole("button", { name: "Add Model" }); + } + + // 'All Model' Tab // + getAllModelsTab(): Locator { + return this.allModelsTab; + } + + // Parametized Locators + getAllModelsTableCellValue(allModelsTableCellValue: string): Locator { + return this.page + .getByRole("cell", { name: allModelsTableCellValue }) + .first(); + } + + // 'Add Model' Tab // + getAddModelTab(): Locator { + return this.addModelTab; + } + + // Parametized Form Locators + /*getProviderComboboxOption(providerComboboxOption: string): Locator { + this.page + .locator("span") + .filter({ hasText: providerComboboxOption }); + }*/ + + fillProviderComboboxBox(providerComboboxText: string) { + this.page + .getByRole("combobox", { name: "* Provider question-circle :" }) + .fill(providerComboboxText); + } + + getLitellmModelNameCombobox(): Locator { + return this.litellmModelNameCombobox; + } + + /*getLitellmModelNameComboboxOption( + litellmModelNameComboboxOption: string + ): Locator { + return this.page + .getByTitle(litellmModelNameComboboxOption, { exact: true }) + .locator("div"); + }*/ + + fillLitellmModelNameCombobox(litellmModelNameComboboxOption: string) { + this.page.locator("#model").fill(litellmModelNameComboboxOption); + } + + getLitellmModelMappingModel(litellmModelMappingModel: string): Locator { + return this.page + .locator("#model_mappings") + .getByText(litellmModelMappingModel); + } + + getLitellmModelMappingModelPublicName( + litellmModelMappingModel: string + ): Locator { + return this.page + .getByRole("row", { name: litellmModelMappingModel }) + .getByTestId("base-input"); + } + + // Non-parametized Form Locators + getProviderCombobox(): Locator { + return this.providerCombobox; + } + + getLitellModelNameCombobox(): Locator { + return this.litellmModelNameCombobox; + } + + getAPIKeyInputBox(provider: string): Locator { + return this.page.getByRole("textbox", { name: `* ${provider} API Key :` }); + } + + getAddModelSubmitButton(): Locator { + return this.addModelSubmitButton; + } +} diff --git a/tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts b/tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts new file mode 100644 index 0000000000..0f7949f2ed --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts @@ -0,0 +1,66 @@ +import type { Page, Locator } from "@playwright/test"; + +export class VirtualKeysPage { + private readonly userButton: Locator; + private readonly logoutButton: Locator; + private readonly createNewKeyButton: Locator; + private readonly ownedByYouRadioButton: Locator; + private readonly keyNameInput: Locator; + private readonly modelInput: Locator; + private readonly createKeyButton: Locator; + private readonly copyAPIKeyButton: Locator; + + constructor(private readonly page: Page) { + this.userButton = this.page.getByRole("button", { name: "User" }); + this.logoutButton = this.page.getByText("Logout"); + this.createNewKeyButton = this.page.getByRole("button", { + name: "+ Create New Key", + }); + this.ownedByYouRadioButton = this.page.getByRole("radio", { name: "You" }); + this.keyNameInput = this.page.getByTestId("base-input"); + this.modelInput = this.page.locator(".ant-select-selection-overflow"); + this.createKeyButton = this.page.getByRole("button", { + name: "Create Key", + }); + this.copyAPIKeyButton = this.page.getByRole("button", { + name: "Copy API Key", + }); + } + + async logout() { + await this.userButton.click(); + await this.logoutButton.click(); + } + + getUserButton(): Locator { + return this.userButton; + } + + getCreateNewKeyButton(): Locator { + return this.createNewKeyButton; + } + + getVirtualKeysTableCellValue(virtualKeysTableCellValue: string): Locator { + return this.page.getByRole("cell", { name: virtualKeysTableCellValue }); + } + + getOwnedByYouRadioButton(): Locator { + return this.ownedByYouRadioButton; + } + + getKeyNameInput(): Locator { + return this.keyNameInput; + } + + getModelInput(): Locator { + return this.modelInput; + } + + getCreateKeyButton(): Locator { + return this.createKeyButton; + } + + getCopyAPIKeyButton(): Locator { + return this.copyAPIKeyButton; + } +} diff --git a/tests/proxy_admin_ui_tests/utils/globalSetup.ts b/tests/proxy_admin_ui_tests/utils/globalSetup.ts new file mode 100644 index 0000000000..fe3c849978 --- /dev/null +++ b/tests/proxy_admin_ui_tests/utils/globalSetup.ts @@ -0,0 +1,8 @@ +import dotenv from "dotenv"; + +export default async function globalSetup() { + dotenv.config({ + //path should be relative to playwright.config.ts + path: "./../../.env", + }); +} diff --git a/tests/proxy_admin_ui_tests/utils/utils.ts b/tests/proxy_admin_ui_tests/utils/utils.ts new file mode 100644 index 0000000000..8261063859 --- /dev/null +++ b/tests/proxy_admin_ui_tests/utils/utils.ts @@ -0,0 +1,14 @@ +// import * as dotenv from 'dotenv'; +// import { config } from "dotenv"; +// import path from "path"; +// config({ path: "./../../../../.env.example" }); + +export function loginDetailsSet(): Boolean { + // console.log(process.env.DATABASE_URL); + // console.log(process.env.UI_PASSWORD); + let loginDetailsSet = false; + if (process.env.UI_USERNAME && process.env.UI_PASSWORD) { + loginDetailsSet = true; + } + return loginDetailsSet; +} From 25b6003ef93b7fd51952644608cd866fb7ff70db Mon Sep 17 00:00:00 2001 From: omnisilica Date: Tue, 1 Apr 2025 14:55:05 -0400 Subject: [PATCH 11/14] Changed Playwright config in order to test all spec files? --- tests/proxy_admin_ui_tests/playwright.config.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/proxy_admin_ui_tests/playwright.config.ts b/tests/proxy_admin_ui_tests/playwright.config.ts index d9e0a67453..b58410acd1 100644 --- a/tests/proxy_admin_ui_tests/playwright.config.ts +++ b/tests/proxy_admin_ui_tests/playwright.config.ts @@ -14,12 +14,7 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ globalSetup: "utils/globalSetup.ts", testDir: "./e2e_ui_tests", - testIgnore: [ - "**/tests/pass_through_tests/**", - "../pass_through_tests/**/*", - "./e2e_ui_tests/login_to_ui.spec.ts", - "./e2e_ui_tests/view_internal_user.spec.ts", - ], + testIgnore: ["**/tests/pass_through_tests/**", "../pass_through_tests/**/*"], testMatch: "**/*.spec.ts", // Only run files ending in .spec.ts /* Run tests in files in parallel */ fullyParallel: true, From b03b7f1a0f9acaa4a512fba869df04d5e4ebcdb2 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Tue, 1 Apr 2025 17:11:21 -0400 Subject: [PATCH 12/14] Slightly re-did view_internal_user e2e test case to factor in edge case --- .../e2e_ui_tests/view_internal_user.spec.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts index de2c8a5720..348d275736 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts @@ -29,7 +29,7 @@ test("view internal user page", async ({ page }) => { // try to click on button // // wait 1-2 seconds - // await page.waitForTimeout(10000); + await page.waitForTimeout(10000); // Test all expected fields are present // number of keys owned by user @@ -39,8 +39,6 @@ test("view internal user page", async ({ page }) => { }) .all(); - await page.pause(); - /* const keysCountArray = await keysBadges.evaluateAll((elements) => { elements.map((el) => { @@ -53,13 +51,17 @@ test("view internal user page", async ({ page }) => { let keysCountArray: number[] = []; for (const element of keysBadges) { - keysCountArray.push( - parseInt((await element.innerText()).split(" ")[0], 10) - ); + if (!((await element.innerText()) === "No Keys")) { + keysCountArray.push( + parseInt((await element.innerText()).split(" ")[0], 10) + ); + } } - const hasNonZeroKeys = keysCountArray.some((count) => count > 0); - expect(hasNonZeroKeys).toBe(true); + if ((await keysBadges[0].first().innerText()) != "No Keys") { + const hasNonZeroKeys = keysCountArray.some((count) => count > 0); + expect(hasNonZeroKeys).toBe(true); + } // test pagination const prevButton = page.locator( From da5884c80df5f269b72021199b10f2cad9efb002 Mon Sep 17 00:00:00 2001 From: omnisilica Date: Sat, 5 Apr 2025 22:16:41 -0400 Subject: [PATCH 13/14] Updated add-model.spec.ts to reflect minor ui change. --- tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts index 02d2425cf4..a761824b9e 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts @@ -77,7 +77,7 @@ providersAndModels["OpenAI"].forEach((model: string) => { // console.log("Clean Up - Delete Model Created"); const modelID = await page .locator("tr.tremor-TableRow-row.h-8") - .nth(1) + .nth(2) .locator(".tremor-Button-text.text-tremor-default") .innerText(); From d492226add89e5111be76e9bfac16db80b825cfe Mon Sep 17 00:00:00 2001 From: omnisilica Date: Fri, 18 Apr 2025 23:31:32 -0400 Subject: [PATCH 14/14] Reverted a couple test files to original implementation from upstream main --- .../e2e_ui_tests/login_to_ui.spec.ts | 34 ++++---- .../e2e_ui_tests/view_internal_user.spec.ts | 78 ++++--------------- 2 files changed, 32 insertions(+), 80 deletions(-) diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts index 4a32c39c6f..f5148bed16 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/login_to_ui.spec.ts @@ -6,17 +6,17 @@ Basic UI Test Click on all the tabs ensure nothing is broken */ -import { test, expect } from "@playwright/test"; +import { test, expect } from '@playwright/test'; -test("admin login test", async ({ page }) => { +test('admin login test', async ({ page }) => { // Go to the specified URL - await page.goto("http://localhost:4000/ui"); + await page.goto('http://localhost:4000/ui'); // Enter "admin" in the username input field - await page.fill('input[name="username"]', "admin"); + await page.fill('input[name="username"]', 'admin'); // Enter "gm" in the password input field - await page.fill('input[name="password"]', "sk-1234"); + await page.fill('input[name="password"]', 'gm'); // Optionally, you can add an assertion to verify the login button is enabled const loginButton = page.locator('input[type="submit"]'); @@ -25,22 +25,20 @@ test("admin login test", async ({ page }) => { // Optionally, you can click the login button to submit the form await loginButton.click(); const tabs = [ - "Virtual Keys", - "Test Key", - "Models", - "Usage", - "Teams", - "Internal User", - "Settings", - "Experimental", - "API Reference", - "Model Hub", + 'Virtual Keys', + 'Test Key', + 'Models', + 'Usage', + 'Teams', + 'Internal User', + 'Settings', + 'Experimental', + 'API Reference', + 'Model Hub' ]; for (const tab of tabs) { - const tabElement = page.locator("span.ant-menu-title-content", { - hasText: tab, - }); + const tabElement = page.locator('span.ant-menu-title-content', { hasText: tab }); await tabElement.click(); } }); diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts index 348d275736..4d27a4a7ce 100644 --- a/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/view_internal_user.spec.ts @@ -2,17 +2,17 @@ Test view internal user page */ -import { test, expect } from "@playwright/test"; +import { test, expect } from '@playwright/test'; -test("view internal user page", async ({ page }) => { +test('view internal user page', async ({ page }) => { // Go to the specified URL - await page.goto("http://localhost:4000/ui"); + await page.goto('http://localhost:4000/ui'); // Enter "admin" in the username input field - await page.fill('input[name="username"]', "admin"); + await page.fill('input[name="username"]', 'admin'); // Enter "gm" in the password input field - await page.fill('input[name="password"]', "sk-1234"); + await page.fill('input[name="password"]', 'gm'); // Optionally, you can add an assertion to verify the login button is enabled const loginButton = page.locator('input[type="submit"]'); @@ -21,72 +21,26 @@ test("view internal user page", async ({ page }) => { // Optionally, you can click the login button to submit the form await loginButton.click(); - const tabElement = page.locator("span.ant-menu-title-content", { - hasText: "Internal User", - }); + const tabElement = page.locator('span.ant-menu-title-content', { hasText: 'Internal User' }); await tabElement.click(); - // try to click on button + // try to click on button // // wait 1-2 seconds await page.waitForTimeout(10000); - // Test all expected fields are present - // number of keys owned by user - const keysBadges = await page - .locator("p.tremor-Badge-text.text-sm.whitespace-nowrap", { - hasText: "Keys", - }) - .all(); + // Test all expected fields are present + // number of keys owned by user + const keysBadges = page.locator('p.tremor-Badge-text.text-sm.whitespace-nowrap', { hasText: 'Keys' }); + const keysCountArray = await keysBadges.evaluateAll(elements => elements.map(el => parseInt(el.textContent.split(' ')[0], 10))); - /* - const keysCountArray = await keysBadges.evaluateAll((elements) => { - elements.map((el) => { - console.log(el); - parseInt(el.textContent.split(" ")[0], 10); - }); - }); - */ - - let keysCountArray: number[] = []; - - for (const element of keysBadges) { - if (!((await element.innerText()) === "No Keys")) { - keysCountArray.push( - parseInt((await element.innerText()).split(" ")[0], 10) - ); - } - } - - if ((await keysBadges[0].first().innerText()) != "No Keys") { - const hasNonZeroKeys = keysCountArray.some((count) => count > 0); - expect(hasNonZeroKeys).toBe(true); - } + const hasNonZeroKeys = keysCountArray.some(count => count > 0); + expect(hasNonZeroKeys).toBe(true); // test pagination - const prevButton = page.locator( - "button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed", - { hasText: "Previous" } - ); + const prevButton = page.locator('button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed', { hasText: 'Previous' }); await expect(prevButton).toBeDisabled(); - let paginationText = await page - .locator(".flex.items-center.space-x-2") - .locator(".text-sm.text-gray-700") - .innerText(); - - let paginationTextContents = paginationText.split(" "); - - let totalPages = parseInt( - paginationTextContents[paginationTextContents.length - 1], - 10 - ); - - if (totalPages > 1) { - const nextButton = page.locator( - "button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed", - { hasText: "Next" } - ); - await expect(nextButton).toBeEnabled(); - } + const nextButton = page.locator('button.px-3.py-1.text-sm.border.rounded-md.hover\\:bg-gray-50.disabled\\:opacity-50.disabled\\:cursor-not-allowed', { hasText: 'Next' }); + await expect(nextButton).toBeEnabled(); });