diff --git a/docs/my-website/docs/proxy/release_cycle.md b/docs/my-website/docs/proxy/release_cycle.md index c5782087f2..947a4ae6b3 100644 --- a/docs/my-website/docs/proxy/release_cycle.md +++ b/docs/my-website/docs/proxy/release_cycle.md @@ -4,17 +4,9 @@ Litellm Proxy has the following release cycle: - `v1.x.x-nightly`: These are releases which pass ci/cd. - `v1.x.x.rc`: These are releases which pass ci/cd + [manual review](https://github.com/BerriAI/litellm/discussions/8495#discussioncomment-12180711). -- `v1.x.x:main-stable`: These are releases which pass ci/cd + manual review + 3 days of production testing. +- `v1.x.x` OR `v1.x.x-stable`: These are releases which pass ci/cd + manual review + 3 days of production testing. -In production, we recommend using the latest `v1.x.x:main-stable` release. +In production, we recommend using the latest `v1.x.x` release. -Follow our release notes [here](https://github.com/BerriAI/litellm/releases). - - -## FAQ - -### Is there a release schedule for LiteLLM stable release? - -Stable releases come out every week (typically Sunday) - +Follow our release notes [here](https://github.com/BerriAI/litellm/releases). \ No newline at end of file diff --git a/tests/proxy_admin_ui_tests/data/providers-and-models.json b/tests/proxy_admin_ui_tests/data/providers-and-models.json new file mode 100644 index 0000000000..95a1e37e8f --- /dev/null +++ b/tests/proxy_admin_ui_tests/data/providers-and-models.json @@ -0,0 +1,26 @@ +{ + "OpenAI": [ + "Custom Model Name (Enter below)", + "All OpenAI Models (Wildcard)", + "omni-moderation-latest", + "omni-moderation-latest-intents", + "omni-moderation-2024-09-26", + "gpt-4", + "gpt-4o", + "gpt-4o-search-preview-2025-03-11", + "gpt-4o-search-preview", + "gpt-4.5-preview" + ], + "Anthropic": [ + "Custom Model Name (Enter below)", + "All Anthropic Models (Wildcard)", + "claude-instant-1", + "claude-instant-1.2", + "claude-2", + "claude-2.1", + "claude-3-haiku-20240307", + "claude-3-5-haiku-20241022", + "claude-3-5-haiku-latest", + "claude-3-opus-latest" + ] +} diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts new file mode 100644 index 0000000000..a761824b9e --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/add-model.spec.ts @@ -0,0 +1,162 @@ +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; +const providersAndModels = JSON.parse( + JSON.stringify(require("./../data/providers-and-models.json")) +); + +providersAndModels["OpenAI"].forEach((model: string) => { + test(`4644_Test_Adding_OpenAI's_${model}_model`, async ({ + loginPage, + dashboardLinks, + modelsPage, + page, + }) => { + const excludeLitellmModelNameDropdownValues = [ + "Custom Model Name (Enter below)", + "All OpenAI Models (Wildcard)", + ]; + if (!excludeLitellmModelNameDropdownValues.includes(model)) { + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); + + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); + + // console.log("4. Selecting OpenAI from 'Provider' dropdown"); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox("OpenAI"); + await modelsPage.getProviderCombobox().press("Enter"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); + + // console.log("5. Selecting model name from 'LiteLLM Model Name(s)' dropdown"); + await modelsPage.getLitellModelNameCombobox().click(); + await modelsPage.getLitellModelNameCombobox().fill(model); + await modelsPage.getLitellModelNameCombobox().press("Enter"); + await expect(modelsPage.getLitellmModelMappingModel(model)).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/05_select-${model}-model.png`,}); + + // console.log("6. Adding API Key"); + await modelsPage.getAPIKeyInputBox("OpenAI").click(); + await modelsPage.getAPIKeyInputBox("OpenAI").fill("sk-1234"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/06_enter-api-key.png`,}); + + // console.log("7. Clicking 'Add Model'"); + await modelsPage.getAddModelSubmitButton().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/07_add-model.png`,}); + + // console.log("8. Navigating to 'All Models'"); + if (model.length > 20) { + model = model.slice(0, 20) + "..."; + } + await modelsPage.getAllModelsTab().click(); + + await expect( + modelsPage.getAllModelsTableCellValue(`openai logo openai`) + ).toBeVisible(); + await expect(modelsPage.getAllModelsTableCellValue(model)).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/08_navigate-to-all-models-tab.png`,}); + + // console.log("Clean Up - Delete Model Created"); + const modelID = await page + .locator("tr.tremor-TableRow-row.h-8") + .nth(2) + .locator(".tremor-Button-text.text-tremor-default") + .innerText(); + + await page.getByRole("button", { name: modelID }).click(); + await page.getByRole("button", { name: "Delete Model" }).click(); + await page.getByRole("button", { name: "Delete", exact: true }).click(); + + // console.log("9. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/09_logout.png`,}); + } + }); +}); + +Object.entries(providersAndModels).forEach(([provider, model]) => { + test(`4644_Test_the_Correct_Dropdown_Shows_When_Adding_${provider}_Models`, async ({ + loginPage, + dashboardLinks, + modelsPage, + page, + }) => { + let username = "admin"; + let password = "sk-1234"; + const excludeLitellmModelNameDropdownValues = [ + "OpenAI", + "OpenAI-Compatible Endpoints (Together AI, etc.)", + "OpenAI Text Completion", + "OpenAI-Compatible Text Completion Models (Together AI, etc.)", + "Anthropic", + ]; + let litellmModelNameDropdownValues: string[] = []; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Navigating to 'Models' page"); + await dashboardLinks.getModelsPageLink().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/02_navigate-to-models-page.png`,}); + + // console.log("3. Selecting 'Add Model' in the header of 'Models' page"); + await modelsPage.getAddModelTab().click(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/03_navigate-to-add-models-tab.png`,}); + + // console.log(`4. Selecting ${model} from 'Provider' dropdown`); + await modelsPage.getProviderCombobox().click(); + modelsPage.fillProviderComboboxBox(provider); + await modelsPage.getProviderCombobox().press("Enter"); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/04_select-openai-provider.png`,}); + + //Scrape ant-selection-option and add to list + await modelsPage.getLitellModelNameCombobox().click(); + const litellmModelOptions = await page + .locator(".rc-virtual-list-holder-inner") + .locator(".ant-select-item-option-content") + .all(); + + for (const element of litellmModelOptions) { + let modelNameDropdownValue = await element.innerText(); + if ( + !excludeLitellmModelNameDropdownValues.includes(modelNameDropdownValue) + ) { + litellmModelNameDropdownValues.push(modelNameDropdownValue); + } + } + litellmModelNameDropdownValues.forEach((element) => { + expect(providersAndModels[provider].includes(element)).toBeTruthy(); + }); + + // console.log("5. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); + }); +}); diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts new file mode 100644 index 0000000000..9e67f06863 --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/create-api-key.spec.ts @@ -0,0 +1,76 @@ +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +test("4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models", async ({ + loginPage, + virtualKeysPage, + page, +}) => { + let username = "admin"; + let password = "sk-1234"; + // let apiKey = ""; + let apiKeyID = ""; + const keyName = "test-key-name-3"; + + if (loginDetailsSet()) { + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + + // console.log("1. Navigating to 'Login' page and logging in"); + await loginPage.goto(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/00_go-to-login-page.png`,}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_test_adding_a_model/openai/${model}/01_dashboard.png`,}); + + // console.log("2. Clicking the '+ Create New Key' button"); + await virtualKeysPage.getCreateNewKeyButton().click(); + + // console.log("3. Clicking the Owned By You radio button."); + await virtualKeysPage.getOwnedByYouRadioButton().check(); + + // console.log("4. Entering a key name in the 'Key Name' input."); + await virtualKeysPage.getKeyNameInput().fill(keyName); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/04_enter-key-name.png`,}); + + // console.log("5. Selecting All Team Models"); + await virtualKeysPage.getModelInput().click(); + await page.getByText("All Team Models").click(); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/05_select-all-team-models.png`,}); + + // console.log("6. Clicking 'Create Key'"); + await virtualKeysPage.getCreateKeyButton().click(); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/06_create-api-key.png`,}); + + // console.log("7. Copying the API key to clipboard"); + /* + await virtualKeysPage.getCopyAPIKeyButton().click(); + apiKey = await page.evaluate(async () => { + return await navigator.clipboard.readText(); + }); + */ + + // console.log("8. Exiting Modal Window"); + await page + .getByRole("dialog") + .filter({ hasText: "Save your KeyPlease save this" }) + .getByLabel("Close", { exact: true }) + .click(); + await expect( + virtualKeysPage.getVirtualKeysTableCellValue(keyName) + ).toBeVisible(); + // await page.screenshot({path: `./test-results/4644_Test_Creating_An_API_Key_for_Self_for_All_Team_Models/08_check-api-key-created.png`,}); + + apiKeyID = await page + .locator("tr.tremor-TableRow-row.h-8") + .locator(".tremor-Button-text.text-tremor-default") + .innerText(); + await page.getByRole("button", { name: apiKeyID }).click(); + await page.getByRole("button", { name: "Delete Key" }).click(); + await page.getByRole("button", { name: "Delete", exact: true }).click(); + + // console.log("9. Logging out"); + await page.getByRole("link", { name: "LiteLLM Brand" }).click(); +}); diff --git a/tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts b/tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts new file mode 100644 index 0000000000..a7de95d575 --- /dev/null +++ b/tests/proxy_admin_ui_tests/e2e_ui_tests/login.spec.ts @@ -0,0 +1,29 @@ +import { VirtualKeysPage } from "../page-object-models/virtual-keys.page"; +import { test, expect } from "./../fixtures/fixtures"; +import { loginDetailsSet } from "./../utils/utils"; + +test("4644_Test_Basic_Sign_in_Flow", async ({ + loginPage, + virtualKeysPage, + page, +}) => { + let username = "admin"; + let password = "sk-1234"; + if (loginDetailsSet()) { + console.log("Login details exist in .env file."); + username = process.env.UI_USERNAME as string; + password = process.env.UI_PASSWORD as string; + } + await loginPage.goto(); + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/navigate-to-login-page.png",}); + + await loginPage.login(username, password); + await expect(page.getByRole("button", { name: "User" })).toBeVisible(); + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/dashboard.png",}); + + await virtualKeysPage.logout(); + await expect( + page.getByRole("heading", { name: "LiteLLM Login" }) + ).toBeVisible(); + // await page.screenshot({path: "./test-results/4644_Test_Basic_Sign_in_Flow/logout.png",}); +}); diff --git a/tests/proxy_admin_ui_tests/fixtures/fixtures.ts b/tests/proxy_admin_ui_tests/fixtures/fixtures.ts new file mode 100644 index 0000000000..b7342d1153 --- /dev/null +++ b/tests/proxy_admin_ui_tests/fixtures/fixtures.ts @@ -0,0 +1,28 @@ +import { DashboardLinks } from "./../page-object-models/dashboard-links"; +import { VirtualKeysPage } from "./../page-object-models/virtual-keys.page"; +import { test as base } from "@playwright/test"; +import { LoginPage } from "../page-object-models/login.page"; +import { ModelsPage } from "../page-object-models/models.page"; + +type Fixtures = { + loginPage: LoginPage; + dashboardLinks: DashboardLinks; + virtualKeysPage: VirtualKeysPage; + modelsPage: ModelsPage; +}; + +export const test = base.extend({ + loginPage: async ({ page }, use) => { + await use(new LoginPage(page)); + }, + dashboardLinks: async ({ page }, use) => { + await use(new DashboardLinks(page)); + }, + virtualKeysPage: async ({ page }, use) => { + await use(new VirtualKeysPage(page)); + }, + modelsPage: async ({ page }, use) => { + await use(new ModelsPage(page)); + }, +}); +export { expect } from "@playwright/test"; diff --git a/tests/proxy_admin_ui_tests/package-lock.json b/tests/proxy_admin_ui_tests/package-lock.json index 3152ee9bfa..0b597aed9c 100644 --- a/tests/proxy_admin_ui_tests/package-lock.json +++ b/tests/proxy_admin_ui_tests/package-lock.json @@ -8,9 +8,12 @@ "name": "proxy_admin_ui_tests", "version": "1.0.0", "license": "ISC", + "dependencies": { + "dotenv": "^16.4.7" + }, "devDependencies": { "@playwright/test": "^1.47.2", - "@types/node": "^22.5.5" + "@types/node": "^22.13.14" } }, "node_modules/@playwright/test": { @@ -29,12 +32,23 @@ } }, "node_modules/@types/node": { - "version": "22.5.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz", - "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==", + "version": "22.13.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.14.tgz", + "integrity": "sha512-Zs/Ollc1SJ8nKUAgc7ivOEdIBM8JAKgrqqUYi2J997JuKO7/tpQC+WCetQ1sypiKCQWHdvdg9wBNpUPEWZae7w==", "dev": true, "dependencies": { - "undici-types": "~6.19.2" + "undici-types": "~6.20.0" + } + }, + "node_modules/dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, "node_modules/fsevents": { @@ -82,9 +96,9 @@ } }, "node_modules/undici-types": { - "version": "6.19.8", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", - "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", "dev": true } } diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index 20dfed7a8a..823f65b8df 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -9,6 +9,9 @@ "license": "ISC", "devDependencies": { "@playwright/test": "^1.47.2", - "@types/node": "^22.5.5" + "@types/node": "^22.13.14" + }, + "dependencies": { + "dotenv": "^16.4.7" } } diff --git a/tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts b/tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts new file mode 100644 index 0000000000..c9b14852ea --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/dashboard-links.ts @@ -0,0 +1,28 @@ +import { Page, Locator } from "@playwright/test"; + +export class DashboardLinks { + private readonly userButton: Locator; + private readonly logoutButton: Locator; + private readonly modelsPageLink: Locator; + + constructor(private readonly page: Page) { + this.userButton = this.page.getByRole("button", { name: "User" }); + this.logoutButton = this.page.getByText("Logout"); + this.modelsPageLink = this.page.getByRole("menuitem", { + name: "block Models", + }); + } + + async logout() { + await this.userButton.click(); + await this.logoutButton.click(); + } + + getUserButton(): Locator { + return this.userButton; + } + + getModelsPageLink(): Locator { + return this.modelsPageLink; + } +} diff --git a/tests/proxy_admin_ui_tests/page-object-models/login.page.ts b/tests/proxy_admin_ui_tests/page-object-models/login.page.ts new file mode 100644 index 0000000000..14df36b5e4 --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/login.page.ts @@ -0,0 +1,27 @@ +import type { Page, Locator } from "@playwright/test"; + +export class LoginPage { + //Locators as fields + private readonly usernameInput: Locator; + private readonly passwordInput: Locator; + private readonly loginSubmit: Locator; + + //Initialize locators in constructor + constructor(private readonly page: Page) { + this.usernameInput = this.page.getByRole("textbox", { name: "Username:" }); + this.passwordInput = this.page.getByRole("textbox", { name: "Password:" }); + this.loginSubmit = this.page.getByRole("button", { name: "Submit" }); + } + + async goto() { + await this.page.goto("/ui"); + } + + async login(username: string, password: string) { + await this.usernameInput.click(); + await this.usernameInput.fill(username); + await this.passwordInput.click(); + await this.passwordInput.fill(password); + await this.loginSubmit.click(); + } +} diff --git a/tests/proxy_admin_ui_tests/page-object-models/models.page.ts b/tests/proxy_admin_ui_tests/page-object-models/models.page.ts new file mode 100644 index 0000000000..f0c0ae3c13 --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/models.page.ts @@ -0,0 +1,114 @@ +import { Page, Locator } from "@playwright/test"; + +export class ModelsPage { + // Models Page Tabs + private readonly allModelsTab: Locator; + private readonly addModelTab: Locator; + // All Models Tab Locators + // Add Model Tab Form Locators + private readonly providerCombobox: Locator; + // *private openaiProviderComboboxOption: Locator; + private readonly litellmModelNameCombobox: Locator; + // *private readonly litellmModelNameComboboxOption page.getByTitle('omni-moderation-latest', { exact: true }).locator('div') + /*private readonly modelMappingPublicNameInput: Locator;*/ + private readonly apiKeyInput: Locator; + private readonly addModelSubmitButton: Locator; + + constructor(private readonly page: Page) { + // Models Page Tabs + this.allModelsTab = this.page.getByRole("tab", { name: "All Models" }); + this.addModelTab = this.page.getByRole("tab", { name: "Add Model" }); + // Add Model Tab Form Locators + this.providerCombobox = this.page.getByRole("combobox", { + name: "* Provider question-circle :", + }); + /**this.openaiProviderComboboxOption = this.page + .locator("span") + .filter({ hasText: "OpenAI" });*/ + this.litellmModelNameCombobox = this.page.locator("#model"); + /*this.modelMappingPublicNameInput = this.page + .getByRole("row", { name: "omni-moderation-latest omni-" }) + .getByTestId("base-input");*/ + this.apiKeyInput = page.getByRole("textbox", { + name: "* API Key question-circle :", + }); + this.addModelSubmitButton = page.getByRole("button", { name: "Add Model" }); + } + + // 'All Model' Tab // + getAllModelsTab(): Locator { + return this.allModelsTab; + } + + // Parametized Locators + getAllModelsTableCellValue(allModelsTableCellValue: string): Locator { + return this.page + .getByRole("cell", { name: allModelsTableCellValue }) + .first(); + } + + // 'Add Model' Tab // + getAddModelTab(): Locator { + return this.addModelTab; + } + + // Parametized Form Locators + /*getProviderComboboxOption(providerComboboxOption: string): Locator { + this.page + .locator("span") + .filter({ hasText: providerComboboxOption }); + }*/ + + fillProviderComboboxBox(providerComboboxText: string) { + this.page + .getByRole("combobox", { name: "* Provider question-circle :" }) + .fill(providerComboboxText); + } + + getLitellmModelNameCombobox(): Locator { + return this.litellmModelNameCombobox; + } + + /*getLitellmModelNameComboboxOption( + litellmModelNameComboboxOption: string + ): Locator { + return this.page + .getByTitle(litellmModelNameComboboxOption, { exact: true }) + .locator("div"); + }*/ + + fillLitellmModelNameCombobox(litellmModelNameComboboxOption: string) { + this.page.locator("#model").fill(litellmModelNameComboboxOption); + } + + getLitellmModelMappingModel(litellmModelMappingModel: string): Locator { + return this.page + .locator("#model_mappings") + .getByText(litellmModelMappingModel); + } + + getLitellmModelMappingModelPublicName( + litellmModelMappingModel: string + ): Locator { + return this.page + .getByRole("row", { name: litellmModelMappingModel }) + .getByTestId("base-input"); + } + + // Non-parametized Form Locators + getProviderCombobox(): Locator { + return this.providerCombobox; + } + + getLitellModelNameCombobox(): Locator { + return this.litellmModelNameCombobox; + } + + getAPIKeyInputBox(provider: string): Locator { + return this.page.getByRole("textbox", { name: `* ${provider} API Key :` }); + } + + getAddModelSubmitButton(): Locator { + return this.addModelSubmitButton; + } +} diff --git a/tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts b/tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts new file mode 100644 index 0000000000..0f7949f2ed --- /dev/null +++ b/tests/proxy_admin_ui_tests/page-object-models/virtual-keys.page.ts @@ -0,0 +1,66 @@ +import type { Page, Locator } from "@playwright/test"; + +export class VirtualKeysPage { + private readonly userButton: Locator; + private readonly logoutButton: Locator; + private readonly createNewKeyButton: Locator; + private readonly ownedByYouRadioButton: Locator; + private readonly keyNameInput: Locator; + private readonly modelInput: Locator; + private readonly createKeyButton: Locator; + private readonly copyAPIKeyButton: Locator; + + constructor(private readonly page: Page) { + this.userButton = this.page.getByRole("button", { name: "User" }); + this.logoutButton = this.page.getByText("Logout"); + this.createNewKeyButton = this.page.getByRole("button", { + name: "+ Create New Key", + }); + this.ownedByYouRadioButton = this.page.getByRole("radio", { name: "You" }); + this.keyNameInput = this.page.getByTestId("base-input"); + this.modelInput = this.page.locator(".ant-select-selection-overflow"); + this.createKeyButton = this.page.getByRole("button", { + name: "Create Key", + }); + this.copyAPIKeyButton = this.page.getByRole("button", { + name: "Copy API Key", + }); + } + + async logout() { + await this.userButton.click(); + await this.logoutButton.click(); + } + + getUserButton(): Locator { + return this.userButton; + } + + getCreateNewKeyButton(): Locator { + return this.createNewKeyButton; + } + + getVirtualKeysTableCellValue(virtualKeysTableCellValue: string): Locator { + return this.page.getByRole("cell", { name: virtualKeysTableCellValue }); + } + + getOwnedByYouRadioButton(): Locator { + return this.ownedByYouRadioButton; + } + + getKeyNameInput(): Locator { + return this.keyNameInput; + } + + getModelInput(): Locator { + return this.modelInput; + } + + getCreateKeyButton(): Locator { + return this.createKeyButton; + } + + getCopyAPIKeyButton(): Locator { + return this.copyAPIKeyButton; + } +} diff --git a/tests/proxy_admin_ui_tests/playwright.config.ts b/tests/proxy_admin_ui_tests/playwright.config.ts index 3be77a319e..b58410acd1 100644 --- a/tests/proxy_admin_ui_tests/playwright.config.ts +++ b/tests/proxy_admin_ui_tests/playwright.config.ts @@ -1,4 +1,4 @@ -import { defineConfig, devices } from '@playwright/test'; +import { defineConfig, devices } from "@playwright/test"; /** * Read environment variables from file. @@ -12,9 +12,10 @@ import { defineConfig, devices } from '@playwright/test'; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - testDir: './e2e_ui_tests', - testIgnore: ['**/tests/pass_through_tests/**', '../pass_through_tests/**/*'], - testMatch: '**/*.spec.ts', // Only run files ending in .spec.ts + globalSetup: "utils/globalSetup.ts", + testDir: "./e2e_ui_tests", + testIgnore: ["**/tests/pass_through_tests/**", "../pass_through_tests/**/*"], + testMatch: "**/*.spec.ts", // Only run files ending in .spec.ts /* Run tests in files in parallel */ fullyParallel: true, /* Fail the build on CI if you accidentally left test.only in the source code. */ @@ -22,33 +23,47 @@ export default defineConfig({ /* Retry on CI only */ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, + // workers: process.env.CI ? 1 : undefined, + workers: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ // baseURL: 'http://127.0.0.1:3000', + baseURL: "http://localhost:4000", /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + trace: "on-first-retry", }, /* Configure projects for major browsers */ projects: [ { - name: 'chromium', - use: { ...devices['Desktop Chrome'] }, + name: "chromium", + use: { + ...devices["Desktop Chrome"], + contextOptions: { + permissions: ["clipboard-read", "clipboard-write"], + }, + }, }, { - name: 'firefox', - use: { ...devices['Desktop Firefox'] }, + name: "firefox", + use: { + ...devices["Desktop Firefox"], + }, }, { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, + name: "webkit", + use: { + ...devices["Desktop Safari"], + contextOptions: { + permissions: ["clipboard-read"], + }, + }, }, /* Test against mobile viewports. */ diff --git a/tests/proxy_admin_ui_tests/utils/globalSetup.ts b/tests/proxy_admin_ui_tests/utils/globalSetup.ts new file mode 100644 index 0000000000..fe3c849978 --- /dev/null +++ b/tests/proxy_admin_ui_tests/utils/globalSetup.ts @@ -0,0 +1,8 @@ +import dotenv from "dotenv"; + +export default async function globalSetup() { + dotenv.config({ + //path should be relative to playwright.config.ts + path: "./../../.env", + }); +} diff --git a/tests/proxy_admin_ui_tests/utils/utils.ts b/tests/proxy_admin_ui_tests/utils/utils.ts new file mode 100644 index 0000000000..8261063859 --- /dev/null +++ b/tests/proxy_admin_ui_tests/utils/utils.ts @@ -0,0 +1,14 @@ +// import * as dotenv from 'dotenv'; +// import { config } from "dotenv"; +// import path from "path"; +// config({ path: "./../../../../.env.example" }); + +export function loginDetailsSet(): Boolean { + // console.log(process.env.DATABASE_URL); + // console.log(process.env.UI_PASSWORD); + let loginDetailsSet = false; + if (process.env.UI_USERNAME && process.env.UI_PASSWORD) { + loginDetailsSet = true; + } + return loginDetailsSet; +}