diff --git a/.circleci/config.yml b/.circleci/config.yml index 1326c1368..3a3fbfd53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2.1 +version: 4.3.4 jobs: local_testing: docker: @@ -189,7 +189,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DOCKER_DB_URL \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ @@ -199,6 +199,7 @@ jobs: -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION=True \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ @@ -209,9 +210,7 @@ jobs: my-app:latest \ --config /app/config.yaml \ --port 4000 \ - --num_workers 8 \ --detailed_debug \ - --run_gunicorn \ - run: name: Install curl and dockerize command: | @@ -226,7 +225,7 @@ jobs: background: true - run: name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 1m + command: dockerize -wait http://localhost:4000 -timeout 5m - run: name: Run tests command: | diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..0a5eb19b6 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,51 @@ +{ + "name": "Python 3.11", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:3.11-bookworm", + // https://github.com/devcontainers/images/tree/main/src/python + // https://mcr.microsoft.com/en-us/product/devcontainers/python/tags + + // "build": { + // "dockerfile": "Dockerfile", + // "context": ".." + // }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + "settings": {}, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "GitHub.copilot", + "GitHub.copilot-chat" + ] + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [4000], + + "containerEnv": { + "LITELLM_LOG": "DEBUG" + }, + + // Use 'portsAttributes' to set default properties for specific forwarded ports. + // More info: https://containers.dev/implementors/json_reference/#port-attributes + "portsAttributes": { + "4000": { + "label": "LiteLLM Server", + "onAutoForward": "notify" + } + }, + + // More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "litellm", + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy" +} \ No newline at end of file diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..f0ced6bed --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,10 @@ +# Add the commit hash of any commit you want to ignore in `git blame` here. +# One commit hash per line. +# +# The GitHub Blame UI will use this file automatically! +# +# Run this command to always ignore formatting commits in `git blame` +# git config blame.ignoreRevsFile .git-blame-ignore-revs + +# Update pydantic code to fix warnings (GH-3600) +876840e9957bc7e9f7d6a2b58c4d7c53dad16481 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 51c578971..b7a164368 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,3 @@ - - - ## Title @@ -18,7 +15,6 @@ 🐛 Bug Fix 🧹 Refactoring 📖 Documentation -💻 Development Environment 🚄 Infrastructure ✅ Test @@ -26,22 +22,8 @@ -## Testing +## [REQUIRED] Testing - Attach a screenshot of any new tests passing locall +If UI changes, send a screenshot/GIF of working UI fixes -## Notes - - - - - -## Pre-Submission Checklist (optional but appreciated): - -- [ ] I have included relevant documentation updates (stored in /docs/my-website) - -## OS Tests (optional but appreciated): - -- [ ] Tested on Windows -- [ ] Tested on MacOS -- [ ] Tested on Linux diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index 9d95c768f..b1a28e069 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -64,6 +64,11 @@ if __name__ == "__main__": ) # Replace with your repository's username and name latest_release = repo.get_latest_release() print("got latest release: ", latest_release) + print(latest_release.title) + print(latest_release.tag_name) + + release_version = latest_release.title + print("latest release body: ", latest_release.body) print("markdown table: ", markdown_table) @@ -74,8 +79,22 @@ if __name__ == "__main__": start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") existing_release_body = latest_release.body[:start_index] + docker_run_command = f""" +\n\n +## Docker Run LiteLLM Proxy + +``` +docker run \\ +-e STORE_MODEL_IN_DB=True \\ +-p 4000:4000 \\ +ghcr.io/berriai/litellm:main-{release_version} +``` + """ + print("docker run command: ", docker_run_command) + new_release_body = ( existing_release_body + + docker_run_command + "\n\n" + "### Don't want to maintain your internal proxy? get in touch 🎉" + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" diff --git a/.gitignore b/.gitignore index abc4ecb0c..b75a92309 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .venv .env +litellm/proxy/myenv/* litellm_uuid.txt __pycache__/ *.pyc @@ -52,3 +53,6 @@ litellm/proxy/_new_secret_config.yaml litellm/proxy/_new_secret_config.yaml litellm/proxy/_super_secret_config.yaml litellm/proxy/_super_secret_config.yaml +litellm/proxy/myenv/bin/activate +litellm/proxy/myenv/bin/Activate.ps1 +myenv/* \ No newline at end of file diff --git a/README.md b/README.md index 9344c0f22..684d5de73 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | +| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | diff --git a/cookbook/liteLLM_clarifai_Demo.ipynb b/cookbook/liteLLM_clarifai_Demo.ipynb new file mode 100644 index 000000000..40ef2fcf9 --- /dev/null +++ b/cookbook/liteLLM_clarifai_Demo.ipynb @@ -0,0 +1,187 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LiteLLM Clarifai \n", + "This notebook walks you through on how to use liteLLM integration of Clarifai and call LLM model from clarifai with response in openAI output format." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-Requisites" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#install necessary packages\n", + "!pip install litellm\n", + "!pip install clarifai" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To obtain Clarifai Personal Access Token follow the steps mentioned in the [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "## Set Clarifai Credentials\n", + "import os\n", + "os.environ[\"CLARIFAI_API_KEY\"]= \"YOUR_CLARIFAI_PAT\" # Clarifai PAT" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Mistral-large" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import litellm\n", + "\n", + "litellm.set_verbose=False" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mistral large response : ModelResponse(id='chatcmpl-6eed494d-7ae2-4870-b9c2-6a64d50a6151', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\"In the grand tapestry of time, where tales unfold,\\nLies the chronicle of ages, a sight to behold.\\nA tale of empires rising, and kings of old,\\nOf civilizations lost, and stories untold.\\n\\nOnce upon a yesterday, in a time so vast,\\nHumans took their first steps, casting shadows in the past.\\nFrom the cradle of mankind, a journey they embarked,\\nThrough stone and bronze and iron, their skills they sharpened and marked.\\n\\nEgyptians built pyramids, reaching for the skies,\\nWhile Greeks sought wisdom, truth, in philosophies that lie.\\nRoman legions marched, their empire to expand,\\nAnd in the East, the Silk Road joined the world, hand in hand.\\n\\nThe Middle Ages came, with knights in shining armor,\\nFeudal lords and serfs, a time of both clamor and calm order.\\nThen Renaissance bloomed, like a flower in the sun,\\nA rebirth of art and science, a new age had begun.\\n\\nAcross the vast oceans, explorers sailed with courage bold,\\nDiscovering new lands, stories of adventure, untold.\\nIndustrial Revolution churned, progress in its wake,\\nMachines and factories, a whole new world to make.\\n\\nTwo World Wars raged, a testament to man's strife,\\nYet from the ashes rose hope, a renewed will for life.\\nInto the modern era, technology took flight,\\nConnecting every corner, bathed in digital light.\\n\\nHistory, a symphony, a melody of time,\\nA testament to human will, resilience so sublime.\\nIn every page, a lesson, in every tale, a guide,\\nFor understanding our past, shapes our future's tide.\", role='assistant'))], created=1713896412, model='https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=13, completion_tokens=338, total_tokens=351))\n" + ] + } + ], + "source": [ + "from litellm import completion\n", + "\n", + "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", + "response=completion(\n", + " model=\"clarifai/mistralai.completion.mistral-large\",\n", + " messages=messages,\n", + " )\n", + "\n", + "print(f\"Mistral large response : {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Claude-2.1 " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Claude-2.1 response : ModelResponse(id='chatcmpl-d126c919-4db4-4aa3-ac8f-7edea41e0b93', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\" Here's a poem I wrote about history:\\n\\nThe Tides of Time\\n\\nThe tides of time ebb and flow,\\nCarrying stories of long ago.\\nFigures and events come into light,\\nShaping the future with all their might.\\n\\nKingdoms rise, empires fall, \\nLeaving traces that echo down every hall.\\nRevolutions bring change with a fiery glow,\\nToppling structures from long ago.\\n\\nExplorers traverse each ocean and land,\\nSeeking treasures they don't understand.\\nWhile artists and writers try to make their mark,\\nHoping their works shine bright in the dark.\\n\\nThe cycle repeats again and again,\\nAs humanity struggles to learn from its pain.\\nThough the players may change on history's stage,\\nThe themes stay the same from age to age.\\n\\nWar and peace, life and death,\\nLove and strife with every breath.\\nThe tides of time continue their dance,\\nAs we join in, by luck or by chance.\\n\\nSo we study the past to light the way forward, \\nHeeding warnings from stories told and heard.\\nThe future unfolds from this unending flow -\\nWhere the tides of time ultimately go.\", role='assistant'))], created=1713896579, model='https://api.clarifai.com/v2/users/anthropic/apps/completion/models/claude-2_1/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=12, completion_tokens=232, total_tokens=244))\n" + ] + } + ], + "source": [ + "from litellm import completion\n", + "\n", + "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", + "response=completion(\n", + " model=\"clarifai/anthropic.completion.claude-2_1\",\n", + " messages=messages,\n", + " )\n", + "\n", + "print(f\"Claude-2.1 response : {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI GPT-4 (Streaming)\n", + "Though clarifai doesn't support streaming, still you can call stream and get the response in standard StreamResponse format of liteLLM" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"In the quiet corners of time's grand hall,\\nLies the tale of rise and fall.\\nFrom ancient ruins to modern sprawl,\\nHistory, the greatest story of them all.\\n\\nEmpires have risen, empires have decayed,\\nThrough the eons, memories have stayed.\\nIn the book of time, history is laid,\\nA tapestry of events, meticulously displayed.\\n\\nThe pyramids of Egypt, standing tall,\\nThe Roman Empire's mighty sprawl.\\nFrom Alexander's conquest, to the Berlin Wall,\\nHistory, a silent witness to it all.\\n\\nIn the shadow of the past we tread,\\nWhere once kings and prophets led.\\nTheir stories in our hearts are spread,\\nEchoes of their words, in our minds are read.\\n\\nBattles fought and victories won,\\nActs of courage under the sun.\\nTales of love, of deeds done,\\nIn history's grand book, they all run.\\n\\nHeroes born, legends made,\\nIn the annals of time, they'll never fade.\\nTheir triumphs and failures all displayed,\\nIn the eternal march of history's parade.\\n\\nThe ink of the past is forever dry,\\nBut its lessons, we cannot deny.\\nIn its stories, truths lie,\\nIn its wisdom, we rely.\\n\\nHistory, a mirror to our past,\\nA guide for the future vast.\\nThrough its lens, we're ever cast,\\nIn the drama of life, forever vast.\", role='assistant', function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n", + "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(content=None, role=None, function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n" + ] + } + ], + "source": [ + "from litellm import completion\n", + "\n", + "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", + "response = completion(\n", + " model=\"clarifai/openai.chat-completion.GPT-4\",\n", + " messages=messages,\n", + " stream=True,\n", + " api_key = \"c75cc032415e45368be331fdd2c06db0\")\n", + "\n", + "for chunk in response:\n", + " print(chunk)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/deploy/azure_resource_manager/azure_marketplace.zip b/deploy/azure_resource_manager/azure_marketplace.zip new file mode 100644 index 000000000..347512586 Binary files /dev/null and b/deploy/azure_resource_manager/azure_marketplace.zip differ diff --git a/deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json b/deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json new file mode 100644 index 000000000..4eba73bdb --- /dev/null +++ b/deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", + "handler": "Microsoft.Azure.CreateUIDef", + "version": "0.1.2-preview", + "parameters": { + "config": { + "isWizard": false, + "basics": { } + }, + "basics": [ ], + "steps": [ ], + "outputs": { }, + "resourceTypes": [ ] + } +} \ No newline at end of file diff --git a/deploy/azure_resource_manager/azure_marketplace/mainTemplate.json b/deploy/azure_resource_manager/azure_marketplace/mainTemplate.json new file mode 100644 index 000000000..114e855bf --- /dev/null +++ b/deploy/azure_resource_manager/azure_marketplace/mainTemplate.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "imageName": { + "type": "string", + "defaultValue": "ghcr.io/berriai/litellm:main-latest" + }, + "containerName": { + "type": "string", + "defaultValue": "litellm-container" + }, + "dnsLabelName": { + "type": "string", + "defaultValue": "litellm" + }, + "portNumber": { + "type": "int", + "defaultValue": 4000 + } + }, + "resources": [ + { + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "2021-03-01", + "name": "[parameters('containerName')]", + "location": "[resourceGroup().location]", + "properties": { + "containers": [ + { + "name": "[parameters('containerName')]", + "properties": { + "image": "[parameters('imageName')]", + "resources": { + "requests": { + "cpu": 1, + "memoryInGB": 2 + } + }, + "ports": [ + { + "port": "[parameters('portNumber')]" + } + ] + } + } + ], + "osType": "Linux", + "restartPolicy": "Always", + "ipAddress": { + "type": "Public", + "ports": [ + { + "protocol": "tcp", + "port": "[parameters('portNumber')]" + } + ], + "dnsNameLabel": "[parameters('dnsLabelName')]" + } + } + } + ] + } \ No newline at end of file diff --git a/deploy/azure_resource_manager/main.bicep b/deploy/azure_resource_manager/main.bicep new file mode 100644 index 000000000..b104cefe1 --- /dev/null +++ b/deploy/azure_resource_manager/main.bicep @@ -0,0 +1,42 @@ +param imageName string = 'ghcr.io/berriai/litellm:main-latest' +param containerName string = 'litellm-container' +param dnsLabelName string = 'litellm' +param portNumber int = 4000 + +resource containerGroupName 'Microsoft.ContainerInstance/containerGroups@2021-03-01' = { + name: containerName + location: resourceGroup().location + properties: { + containers: [ + { + name: containerName + properties: { + image: imageName + resources: { + requests: { + cpu: 1 + memoryInGB: 2 + } + } + ports: [ + { + port: portNumber + } + ] + } + } + ] + osType: 'Linux' + restartPolicy: 'Always' + ipAddress: { + type: 'Public' + ports: [ + { + protocol: 'tcp' + port: portNumber + } + ] + dnsNameLabel: dnsLabelName + } + } +} diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index 628b76a3c..7f68acf88 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -24,7 +24,7 @@ version: 0.2.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: v1.24.5 +appVersion: v1.35.38 dependencies: - name: "postgresql" diff --git a/docs/my-website/docs/completion/batching.md b/docs/my-website/docs/completion/batching.md index 05683b3dd..09f59f743 100644 --- a/docs/my-website/docs/completion/batching.md +++ b/docs/my-website/docs/completion/batching.md @@ -4,6 +4,12 @@ LiteLLM allows you to: * Send 1 completion call to many models: Return Fastest Response * Send 1 completion call to many models: Return All Responses +:::info + +Trying to do batch completion on LiteLLM Proxy ? Go here: https://docs.litellm.ai/docs/proxy/user_keys#beta-batch-completions---pass-model-as-list + +::: + ## Send multiple completion calls to 1 model In the batch_completion method, you provide a list of `messages` where each sub-list of messages is passed to `litellm.completion()`, allowing you to process multiple prompts efficiently in a single API call. diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index fd5594610..ba01dd9d8 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -37,11 +37,12 @@ print(response) # ["max_tokens", "tools", "tool_choice", "stream"] This is a list of openai params we translate across providers. -This list is constantly being updated. +Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider | Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--| -|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ | ✅ | +|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | ✅ | ✅ | ✅ | |OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | |Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ | |Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | @@ -83,8 +84,9 @@ def completion( top_p: Optional[float] = None, n: Optional[int] = None, stream: Optional[bool] = None, + stream_options: Optional[dict] = None, stop=None, - max_tokens: Optional[float] = None, + max_tokens: Optional[int] = None, presence_penalty: Optional[float] = None, frequency_penalty: Optional[float] = None, logit_bias: Optional[dict] = None, @@ -139,6 +141,10 @@ def completion( - `stream`: *boolean or null (optional)* - If set to true, it sends partial message deltas. Tokens will be sent as they become available, with the stream terminated by a [DONE] message. +- `stream_options` *dict or null (optional)* - Options for streaming response. Only set this when you set `stream: true` + + - `include_usage` *boolean (optional)* - If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value. + - `stop`: *string/ array/ null (optional)* - Up to 4 sequences where the API will stop generating further tokens. - `max_tokens`: *integer (optional)* - The maximum number of tokens to generate in the chat completion. diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 52cfee7ac..ebf7a29eb 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -320,8 +320,6 @@ from litellm import embedding litellm.vertex_project = "hardy-device-38811" # Your Project ID litellm.vertex_location = "us-central1" # proj location - -os.environ['VOYAGE_API_KEY'] = "" response = embedding( model="vertex_ai/textembedding-gecko", input=["good morning from litellm"], diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 68091fe2e..382ba8b28 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -17,6 +17,14 @@ This covers: - ✅ [**JWT-Auth**](../docs/proxy/token_auth.md) +## [COMING SOON] AWS Marketplace Support + +Deploy managed LiteLLM Proxy within your VPC. + +Includes all enterprise features. + +[**Get early access**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + ## Frequently Asked Questions ### What topics does Professional support cover and what SLAs do you offer? diff --git a/docs/my-website/docs/exception_mapping.md b/docs/my-website/docs/exception_mapping.md index db17fb093..5e6006ebe 100644 --- a/docs/my-website/docs/exception_mapping.md +++ b/docs/my-website/docs/exception_mapping.md @@ -13,7 +13,7 @@ LiteLLM maps exceptions across all providers to their OpenAI counterparts. | >=500 | InternalServerError | | N/A | ContextWindowExceededError| | 400 | ContentPolicyViolationError| -| N/A | APIConnectionError | +| 500 | APIConnectionError | Base case we return APIConnectionError @@ -74,6 +74,28 @@ except Exception as e: ``` +## Usage - Should you retry exception? + +``` +import litellm +import openai + +try: + response = litellm.completion( + model="gpt-4", + messages=[ + { + "role": "user", + "content": "hello, write a 20 pageg essay" + } + ], + timeout=0.01, # this will raise a timeout exception + ) +except openai.APITimeoutError as e: + should_retry = litellm._should_retry(e.status_code) + print(f"should_retry: {should_retry}") +``` + ## Details To see how it's implemented - [check out the code](https://github.com/BerriAI/litellm/blob/a42c197e5a6de56ea576c73715e6c7c6b19fa249/litellm/utils.py#L1217) @@ -84,23 +106,37 @@ To see how it's implemented - [check out the code](https://github.com/BerriAI/li ## Custom mapping list -Base case - we return the original exception. +Base case - we return `litellm.APIConnectionError` exception (inherits from openai's APIConnectionError exception). -| | ContextWindowExceededError | AuthenticationError | InvalidRequestError | RateLimitError | ServiceUnavailableError | -|---------------|----------------------------|---------------------|---------------------|---------------|-------------------------| -| Anthropic | ✅ | ✅ | ✅ | ✅ | | -| OpenAI | ✅ | ✅ |✅ |✅ |✅| -| Azure OpenAI | ✅ | ✅ |✅ |✅ |✅| -| Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | -| Cohere | ✅ | ✅ | ✅ | ✅ | ✅ | -| Huggingface | ✅ | ✅ | ✅ | ✅ | | -| Openrouter | ✅ | ✅ | ✅ | ✅ | | -| AI21 | ✅ | ✅ | ✅ | ✅ | | -| VertexAI | | |✅ | | | -| Bedrock | | |✅ | | | -| Sagemaker | | |✅ | | | -| TogetherAI | ✅ | ✅ | ✅ | ✅ | | -| AlephAlpha | ✅ | ✅ | ✅ | ✅ | ✅ | +| custom_llm_provider | Timeout | ContextWindowExceededError | BadRequestError | NotFoundError | ContentPolicyViolationError | AuthenticationError | APIError | RateLimitError | ServiceUnavailableError | PermissionDeniedError | UnprocessableEntityError | +|----------------------------|---------|----------------------------|------------------|---------------|-----------------------------|---------------------|----------|----------------|-------------------------|-----------------------|-------------------------| +| openai | ✓ | ✓ | ✓ | | ✓ | ✓ | | | | | | +| watsonx | | | | | | | |✓| | | | +| text-completion-openai | ✓ | ✓ | ✓ | | ✓ | ✓ | | | | | | +| custom_openai | ✓ | ✓ | ✓ | | ✓ | ✓ | | | | | | +| openai_compatible_providers| ✓ | ✓ | ✓ | | ✓ | ✓ | | | | | | +| anthropic | ✓ | ✓ | ✓ | ✓ | | ✓ | | | ✓ | ✓ | | +| replicate | ✓ | ✓ | ✓ | ✓ | | ✓ | | ✓ | ✓ | | | +| bedrock | ✓ | ✓ | ✓ | ✓ | | ✓ | | ✓ | ✓ | ✓ | | +| sagemaker | | ✓ | ✓ | | | | | | | | | +| vertex_ai | ✓ | | ✓ | | | | ✓ | | | | ✓ | +| palm | ✓ | ✓ | | | | | ✓ | | | | | +| gemini | ✓ | ✓ | | | | | ✓ | | | | | +| cloudflare | | | ✓ | | | ✓ | | | | | | +| cohere | | ✓ | ✓ | | | ✓ | | | ✓ | | | +| cohere_chat | | ✓ | ✓ | | | ✓ | | | ✓ | | | +| huggingface | ✓ | ✓ | ✓ | | | ✓ | | ✓ | ✓ | | | +| ai21 | ✓ | ✓ | ✓ | ✓ | | ✓ | | ✓ | | | | +| nlp_cloud | ✓ | ✓ | ✓ | | | ✓ | ✓ | ✓ | ✓ | | | +| together_ai | ✓ | ✓ | ✓ | | | ✓ | | | | | | +| aleph_alpha | | | ✓ | | | ✓ | | | | | | +| ollama | ✓ | | ✓ | | | | | | ✓ | | | +| ollama_chat | ✓ | | ✓ | | | | | | ✓ | | | +| vllm | | | | | | ✓ | ✓ | | | | | +| azure | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | | ✓ | | | + +- "✓" indicates that the specified `custom_llm_provider` can raise the corresponding exception. +- Empty cells indicate the lack of association or that the provider does not raise that particular exception type as indicated by the function. > For a deeper understanding of these exceptions, you can check out [this](https://github.com/BerriAI/litellm/blob/d7e58d13bf9ba9edbab2ab2f096f3de7547f35fa/litellm/utils.py#L1544) implementation for additional insights. diff --git a/docs/my-website/docs/hosted.md b/docs/my-website/docs/hosted.md index 9be6e775d..92940e858 100644 --- a/docs/my-website/docs/hosted.md +++ b/docs/my-website/docs/hosted.md @@ -46,4 +46,13 @@ Pricing is based on usage. We can figure out a price that works for your team, o -#### [**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) \ No newline at end of file +#### [**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +## Feature List + +- Easy way to add/remove models +- 100% uptime even when models are added/removed +- custom callback webhooks +- your domain name with HTTPS +- Ability to create/delete User API keys +- Reasonable set monthly cost \ No newline at end of file diff --git a/docs/my-website/docs/langchain/langchain.md b/docs/my-website/docs/langchain/langchain.md index cc12767b8..efa6b2925 100644 --- a/docs/my-website/docs/langchain/langchain.md +++ b/docs/my-website/docs/langchain/langchain.md @@ -14,14 +14,14 @@ import TabItem from '@theme/TabItem'; ```python import os -from langchain.chat_models import ChatLiteLLM -from langchain.prompts.chat import ( +from langchain_community.chat_models import ChatLiteLLM +from langchain_core.prompts import ( ChatPromptTemplate, SystemMessagePromptTemplate, AIMessagePromptTemplate, HumanMessagePromptTemplate, ) -from langchain.schema import AIMessage, HumanMessage, SystemMessage +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage os.environ['OPENAI_API_KEY'] = "" chat = ChatLiteLLM(model="gpt-3.5-turbo") @@ -30,7 +30,7 @@ messages = [ content="what model are you" ) ] -chat(messages) +chat.invoke(messages) ``` @@ -39,14 +39,14 @@ chat(messages) ```python import os -from langchain.chat_models import ChatLiteLLM -from langchain.prompts.chat import ( +from langchain_community.chat_models import ChatLiteLLM +from langchain_core.prompts import ( ChatPromptTemplate, SystemMessagePromptTemplate, AIMessagePromptTemplate, HumanMessagePromptTemplate, ) -from langchain.schema import AIMessage, HumanMessage, SystemMessage +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage os.environ['ANTHROPIC_API_KEY'] = "" chat = ChatLiteLLM(model="claude-2", temperature=0.3) @@ -55,7 +55,7 @@ messages = [ content="what model are you" ) ] -chat(messages) +chat.invoke(messages) ``` @@ -64,14 +64,14 @@ chat(messages) ```python import os -from langchain.chat_models import ChatLiteLLM -from langchain.prompts.chat import ( +from langchain_community.chat_models import ChatLiteLLM +from langchain_core.prompts.chat import ( ChatPromptTemplate, SystemMessagePromptTemplate, AIMessagePromptTemplate, HumanMessagePromptTemplate, ) -from langchain.schema import AIMessage, HumanMessage, SystemMessage +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage os.environ['REPLICATE_API_TOKEN'] = "" chat = ChatLiteLLM(model="replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1") @@ -80,7 +80,7 @@ messages = [ content="what model are you?" ) ] -chat(messages) +chat.invoke(messages) ``` @@ -89,14 +89,14 @@ chat(messages) ```python import os -from langchain.chat_models import ChatLiteLLM -from langchain.prompts.chat import ( +from langchain_community.chat_models import ChatLiteLLM +from langchain_core.prompts import ( ChatPromptTemplate, SystemMessagePromptTemplate, AIMessagePromptTemplate, HumanMessagePromptTemplate, ) -from langchain.schema import AIMessage, HumanMessage, SystemMessage +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage os.environ['COHERE_API_KEY'] = "" chat = ChatLiteLLM(model="command-nightly") @@ -105,32 +105,9 @@ messages = [ content="what model are you?" ) ] -chat(messages) +chat.invoke(messages) ``` - - - -```python -import os -from langchain.chat_models import ChatLiteLLM -from langchain.prompts.chat import ( - ChatPromptTemplate, - SystemMessagePromptTemplate, - AIMessagePromptTemplate, - HumanMessagePromptTemplate, -) -from langchain.schema import AIMessage, HumanMessage, SystemMessage - -os.environ['PALM_API_KEY'] = "" -chat = ChatLiteLLM(model="palm/chat-bison") -messages = [ - HumanMessage( - content="what model are you?" - ) -] -chat(messages) -``` diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index bf62ee9bc..508ba63cd 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -94,9 +94,10 @@ print(response) ``` -### Set Custom Trace ID, Trace User ID and Tags +### Set Custom Trace ID, Trace User ID, Trace Metadata, Trace Version, Trace Release and Tags + +Pass `trace_id`, `trace_user_id`, `trace_metadata`, `trace_version`, `trace_release`, `tags` in `metadata` -Pass `trace_id`, `trace_user_id` in `metadata` ```python import litellm @@ -121,12 +122,21 @@ response = completion( metadata={ "generation_name": "ishaan-test-generation", # set langfuse Generation Name "generation_id": "gen-id22", # set langfuse Generation ID + "version": "test-generation-version" # set langfuse Generation Version "trace_user_id": "user-id2", # set langfuse Trace User ID "session_id": "session-1", # set langfuse Session ID - "tags": ["tag1", "tag2"] # set langfuse Tags + "tags": ["tag1", "tag2"], # set langfuse Tags "trace_id": "trace-id22", # set langfuse Trace ID + "trace_metadata": {"key": "value"}, # set langfuse Trace Metadata + "trace_version": "test-trace-version", # set langfuse Trace Version (if not set, defaults to Generation Version) + "trace_release": "test-trace-release", # set langfuse Trace Release ### OR ### - "existing_trace_id": "trace-id22", # if generation is continuation of past trace. This prevents default behaviour of setting a trace name + "existing_trace_id": "trace-id22", # if generation is continuation of past trace. This prevents default behaviour of setting a trace name + ### OR enforce that certain fields are trace overwritten in the trace during the continuation ### + "existing_trace_id": "trace-id22", + "trace_metadata": {"key": "updated_trace_value"}, # The new value to use for the langfuse Trace Metadata + "update_trace_keys": ["input", "output", "trace_metadata"], # Updates the trace input & output to be this generations input & output also updates the Trace Metadata to match the passed in value + "debug_langfuse": True, # Will log the exact metadata sent to litellm for the trace/generation as `metadata_passed_to_litellm` }, ) @@ -134,6 +144,38 @@ print(response) ``` +### Trace & Generation Parameters + +#### Trace Specific Parameters + +* `trace_id` - Identifier for the trace, must use `existing_trace_id` instead or in conjunction with `trace_id` if this is an existing trace, auto-generated by default +* `trace_name` - Name of the trace, auto-generated by default +* `session_id` - Session identifier for the trace, defaults to `None` +* `trace_version` - Version for the trace, defaults to value for `version` +* `trace_release` - Release for the trace, defaults to `None` +* `trace_metadata` - Metadata for the trace, defaults to `None` +* `trace_user_id` - User identifier for the trace, defaults to completion argument `user` +* `tags` - Tags for the trace, defeaults to `None` + +##### Updatable Parameters on Continuation + +The following parameters can be updated on a continuation of a trace by passing in the following values into the `update_trace_keys` in the metadata of the completion. + +* `input` - Will set the traces input to be the input of this latest generation +* `output` - Will set the traces output to be the output of this generation +* `trace_version` - Will set the trace version to be the provided value (To use the latest generations version instead, use `version`) +* `trace_release` - Will set the trace release to be the provided value +* `trace_metadata` - Will set the trace metadata to the provided value +* `trace_user_id` - Will set the trace user id to the provided value + +#### Generation Specific Parameters + +* `generation_id` - Identifier for the generation, auto-generated by default +* `generation_name` - Identifier for the generation, auto-generated by default +* `prompt` - Langfuse prompt object used for the generation, defaults to None + +Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation. + ### Use LangChain ChatLiteLLM + Langfuse Pass `trace_user_id`, `session_id` in model_kwargs ```python @@ -171,8 +213,20 @@ chat(messages) ## Redacting Messages, Response Content from Langfuse Logging +### Redact Messages and Responses from all Langfuse Logging + Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. +### Redact Messages and Responses from specific Langfuse Logging + +In the metadata typically passed for text completion or embedding calls you can set specific keys to mask the messages and responses for this call. + +Setting `mask_input` to `True` will mask the input from being logged for this call + +Setting `mask_output` to `True` will make the output from being logged for this call. + +Be aware that if you are continuing an existing trace, and you set `update_trace_keys` to include either `input` or `output` and you set the corresponding `mask_input` or `mask_output`, then that trace will have its existing input and/or output replaced with a redacted message. + ## Troubleshooting & Errors ### Data not getting logged to Langfuse ? - Ensure you're on the latest version of langfuse `pip install langfuse -U`. The latest version allows litellm to log JSON input/outputs to langfuse diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 590ffc423..147c12e65 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -535,7 +535,8 @@ print(response) | Model Name | Function Call | |----------------------|---------------------------------------------| -| Titan Embeddings - G1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | +| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | +| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | | Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | | Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | diff --git a/docs/my-website/docs/providers/clarifai.md b/docs/my-website/docs/providers/clarifai.md new file mode 100644 index 000000000..acc8c54be --- /dev/null +++ b/docs/my-website/docs/providers/clarifai.md @@ -0,0 +1,177 @@ + +# Clarifai +Anthropic, OpenAI, Mistral, Llama and Gemini LLMs are Supported on Clarifai. + +## Pre-Requisites + +`pip install clarifai` + +`pip install litellm` + +## Required Environment Variables +To obtain your Clarifai Personal access token follow this [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/). Optionally the PAT can also be passed in `completion` function. + +```python +os.environ["CALRIFAI_API_KEY"] = "YOUR_CLARIFAI_PAT" # CLARIFAI_PAT +``` + +## Usage + +```python +import os +from litellm import completion + +os.environ["CLARIFAI_API_KEY"] = "" + +response = completion( + model="clarifai/mistralai.completion.mistral-large", + messages=[{ "content": "Tell me a joke about physics?","role": "user"}] +) +``` + +**Output** +```json +{ + "id": "chatcmpl-572701ee-9ab2-411c-ac75-46c1ba18e781", + "choices": [ + { + "finish_reason": "stop", + "index": 1, + "message": { + "content": "Sure, here's a physics joke for you:\n\nWhy can't you trust an atom?\n\nBecause they make up everything!", + "role": "assistant" + } + } + ], + "created": 1714410197, + "model": "https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "prompt_tokens": 14, + "completion_tokens": 24, + "total_tokens": 38 + } + } +``` + +## Clarifai models +liteLLM supports non-streaming requests to all models on [Clarifai community](https://clarifai.com/explore/models?filterData=%5B%7B%22field%22%3A%22use_cases%22%2C%22value%22%3A%5B%22llm%22%5D%7D%5D&page=1&perPage=24) + +Example Usage - Note: liteLLM supports all models deployed on Clarifai + +## Llama LLMs +| Model Name | Function Call | +---------------------------|---------------------------------| +| clarifai/meta.Llama-2.llama2-7b-chat | `completion('clarifai/meta.Llama-2.llama2-7b-chat', messages)` +| clarifai/meta.Llama-2.llama2-13b-chat | `completion('clarifai/meta.Llama-2.llama2-13b-chat', messages)` +| clarifai/meta.Llama-2.llama2-70b-chat | `completion('clarifai/meta.Llama-2.llama2-70b-chat', messages)` | +| clarifai/meta.Llama-2.codeLlama-70b-Python | `completion('clarifai/meta.Llama-2.codeLlama-70b-Python', messages)`| +| clarifai/meta.Llama-2.codeLlama-70b-Instruct | `completion('clarifai/meta.Llama-2.codeLlama-70b-Instruct', messages)` | + +## Mistal LLMs +| Model Name | Function Call | +|---------------------------------------------|------------------------------------------------------------------------| +| clarifai/mistralai.completion.mixtral-8x22B | `completion('clarifai/mistralai.completion.mixtral-8x22B', messages)` | +| clarifai/mistralai.completion.mistral-large | `completion('clarifai/mistralai.completion.mistral-large', messages)` | +| clarifai/mistralai.completion.mistral-medium | `completion('clarifai/mistralai.completion.mistral-medium', messages)` | +| clarifai/mistralai.completion.mistral-small | `completion('clarifai/mistralai.completion.mistral-small', messages)` | +| clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1 | `completion('clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1', messages)` +| clarifai/mistralai.completion.mistral-7B-OpenOrca | `completion('clarifai/mistralai.completion.mistral-7B-OpenOrca', messages)` | +| clarifai/mistralai.completion.openHermes-2-mistral-7B | `completion('clarifai/mistralai.completion.openHermes-2-mistral-7B', messages)` | + + +## Jurassic LLMs +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/ai21.complete.Jurassic2-Grande | `completion('clarifai/ai21.complete.Jurassic2-Grande', messages)` | +| clarifai/ai21.complete.Jurassic2-Grande-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Grande-Instruct', messages)` | +| clarifai/ai21.complete.Jurassic2-Jumbo-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Jumbo-Instruct', messages)` | +| clarifai/ai21.complete.Jurassic2-Jumbo | `completion('clarifai/ai21.complete.Jurassic2-Jumbo', messages)` | +| clarifai/ai21.complete.Jurassic2-Large | `completion('clarifai/ai21.complete.Jurassic2-Large', messages)` | + +## Wizard LLMs + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/wizardlm.generate.wizardCoder-Python-34B | `completion('clarifai/wizardlm.generate.wizardCoder-Python-34B', messages)` | +| clarifai/wizardlm.generate.wizardLM-70B | `completion('clarifai/wizardlm.generate.wizardLM-70B', messages)` | +| clarifai/wizardlm.generate.wizardLM-13B | `completion('clarifai/wizardlm.generate.wizardLM-13B', messages)` | +| clarifai/wizardlm.generate.wizardCoder-15B | `completion('clarifai/wizardlm.generate.wizardCoder-15B', messages)` | + +## Anthropic models + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/anthropic.completion.claude-v1 | `completion('clarifai/anthropic.completion.claude-v1', messages)` | +| clarifai/anthropic.completion.claude-instant-1_2 | `completion('clarifai/anthropic.completion.claude-instant-1_2', messages)` | +| clarifai/anthropic.completion.claude-instant | `completion('clarifai/anthropic.completion.claude-instant', messages)` | +| clarifai/anthropic.completion.claude-v2 | `completion('clarifai/anthropic.completion.claude-v2', messages)` | +| clarifai/anthropic.completion.claude-2_1 | `completion('clarifai/anthropic.completion.claude-2_1', messages)` | +| clarifai/anthropic.completion.claude-3-opus | `completion('clarifai/anthropic.completion.claude-3-opus', messages)` | +| clarifai/anthropic.completion.claude-3-sonnet | `completion('clarifai/anthropic.completion.claude-3-sonnet', messages)` | + +## OpenAI GPT LLMs + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/openai.chat-completion.GPT-4 | `completion('clarifai/openai.chat-completion.GPT-4', messages)` | +| clarifai/openai.chat-completion.GPT-3_5-turbo | `completion('clarifai/openai.chat-completion.GPT-3_5-turbo', messages)` | +| clarifai/openai.chat-completion.gpt-4-turbo | `completion('clarifai/openai.chat-completion.gpt-4-turbo', messages)` | +| clarifai/openai.completion.gpt-3_5-turbo-instruct | `completion('clarifai/openai.completion.gpt-3_5-turbo-instruct', messages)` | + +## GCP LLMs + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/gcp.generate.gemini-1_5-pro | `completion('clarifai/gcp.generate.gemini-1_5-pro', messages)` | +| clarifai/gcp.generate.imagen-2 | `completion('clarifai/gcp.generate.imagen-2', messages)` | +| clarifai/gcp.generate.code-gecko | `completion('clarifai/gcp.generate.code-gecko', messages)` | +| clarifai/gcp.generate.code-bison | `completion('clarifai/gcp.generate.code-bison', messages)` | +| clarifai/gcp.generate.text-bison | `completion('clarifai/gcp.generate.text-bison', messages)` | +| clarifai/gcp.generate.gemma-2b-it | `completion('clarifai/gcp.generate.gemma-2b-it', messages)` | +| clarifai/gcp.generate.gemma-7b-it | `completion('clarifai/gcp.generate.gemma-7b-it', messages)` | +| clarifai/gcp.generate.gemini-pro | `completion('clarifai/gcp.generate.gemini-pro', messages)` | +| clarifai/gcp.generate.gemma-1_1-7b-it | `completion('clarifai/gcp.generate.gemma-1_1-7b-it', messages)` | + +## Cohere LLMs +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/cohere.generate.cohere-generate-command | `completion('clarifai/cohere.generate.cohere-generate-command', messages)` | + clarifai/cohere.generate.command-r-plus' | `completion('clarifai/clarifai/cohere.generate.command-r-plus', messages)`| + +## Databricks LLMs + +| Model Name | Function Call | +|---------------------------------------------------|---------------------------------------------------------------------| +| clarifai/databricks.drbx.dbrx-instruct | `completion('clarifai/databricks.drbx.dbrx-instruct', messages)` | +| clarifai/databricks.Dolly-v2.dolly-v2-12b | `completion('clarifai/databricks.Dolly-v2.dolly-v2-12b', messages)`| + +## Microsoft LLMs + +| Model Name | Function Call | +|---------------------------------------------------|---------------------------------------------------------------------| +| clarifai/microsoft.text-generation.phi-2 | `completion('clarifai/microsoft.text-generation.phi-2', messages)` | +| clarifai/microsoft.text-generation.phi-1_5 | `completion('clarifai/microsoft.text-generation.phi-1_5', messages)`| + +## Salesforce models + +| Model Name | Function Call | +|-----------------------------------------------------------|-------------------------------------------------------------------------------| +| clarifai/salesforce.blip.general-english-image-caption-blip-2 | `completion('clarifai/salesforce.blip.general-english-image-caption-blip-2', messages)` | +| clarifai/salesforce.xgen.xgen-7b-8k-instruct | `completion('clarifai/salesforce.xgen.xgen-7b-8k-instruct', messages)` | + + +## Other Top performing LLMs + +| Model Name | Function Call | +|---------------------------------------------------|---------------------------------------------------------------------| +| clarifai/deci.decilm.deciLM-7B-instruct | `completion('clarifai/deci.decilm.deciLM-7B-instruct', messages)` | +| clarifai/upstage.solar.solar-10_7b-instruct | `completion('clarifai/upstage.solar.solar-10_7b-instruct', messages)` | +| clarifai/openchat.openchat.openchat-3_5-1210 | `completion('clarifai/openchat.openchat.openchat-3_5-1210', messages)` | +| clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B | `completion('clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B', messages)` | +| clarifai/fblgit.una-cybertron.una-cybertron-7b-v2 | `completion('clarifai/fblgit.una-cybertron.una-cybertron-7b-v2', messages)` | +| clarifai/tiiuae.falcon.falcon-40b-instruct | `completion('clarifai/tiiuae.falcon.falcon-40b-instruct', messages)` | +| clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat | `completion('clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat', messages)` | +| clarifai/bigcode.code.StarCoder | `completion('clarifai/bigcode.code.StarCoder', messages)` | +| clarifai/mosaicml.mpt.mpt-7b-instruct | `completion('clarifai/mosaicml.mpt.mpt-7b-instruct', messages)` | diff --git a/docs/my-website/docs/providers/deepseek.md b/docs/my-website/docs/providers/deepseek.md new file mode 100644 index 000000000..678561eca --- /dev/null +++ b/docs/my-website/docs/providers/deepseek.md @@ -0,0 +1,54 @@ +# Deepseek +https://deepseek.com/ + +**We support ALL Deepseek models, just set `deepseek/` as a prefix when sending completion requests** + +## API Key +```python +# env variable +os.environ['DEEPSEEK_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['DEEPSEEK_API_KEY'] = "" +response = completion( + model="deepseek/deepseek-chat", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['DEEPSEEK_API_KEY'] = "" +response = completion( + model="deepseek/deepseek-chat", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + + +## Supported Models - ALL Deepseek Models Supported! +We support ALL Deepseek models, just set `deepseek/` as a prefix when sending completion requests + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| deepseek-chat | `completion(model="deepseek/deepseek-chat", messages)` | +| deepseek-coder | `completion(model="deepseek/deepseek-chat", messages)` | + + diff --git a/docs/my-website/docs/providers/huggingface.md b/docs/my-website/docs/providers/huggingface.md index f8ebadfcf..35befd3e2 100644 --- a/docs/my-website/docs/providers/huggingface.md +++ b/docs/my-website/docs/providers/huggingface.md @@ -21,6 +21,11 @@ This is done by adding the "huggingface/" prefix to `model`, example `completion +By default, LiteLLM will assume a huggingface call follows the TGI format. + + + + ```python import os from litellm import completion @@ -40,9 +45,58 @@ response = completion( print(response) ``` + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: wizard-coder + litellm_params: + model: huggingface/WizardLM/WizardCoder-Python-34B-V1.0 + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "wizard-coder", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + +Append `conversational` to the model name + +e.g. `huggingface/conversational/` + + + + ```python import os from litellm import completion @@ -54,7 +108,7 @@ messages = [{ "content": "There's a llama in my garden 😱 What should I do?"," # e.g. Call 'facebook/blenderbot-400M-distill' hosted on HF Inference endpoints response = completion( - model="huggingface/facebook/blenderbot-400M-distill", + model="huggingface/conversational/facebook/blenderbot-400M-distill", messages=messages, api_base="https://my-endpoint.huggingface.cloud" ) @@ -62,7 +116,123 @@ response = completion( print(response) ``` - + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: blenderbot + litellm_params: + model: huggingface/conversational/facebook/blenderbot-400M-distill + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "blenderbot", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + + + + +Append `text-classification` to the model name + +e.g. `huggingface/text-classification/` + + + + +```python +import os +from litellm import completion + +# [OPTIONAL] set env var +os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" + +messages = [{ "content": "I like you, I love you!","role": "user"}] + +# e.g. Call 'shahrukhx01/question-vs-statement-classifier' hosted on HF Inference endpoints +response = completion( + model="huggingface/text-classification/shahrukhx01/question-vs-statement-classifier", + messages=messages, + api_base="https://my-endpoint.endpoints.huggingface.cloud", +) + +print(response) +``` + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: bert-classifier + litellm_params: + model: huggingface/text-classification/shahrukhx01/question-vs-statement-classifier + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "bert-classifier", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + + + + +Append `text-generation` to the model name + +e.g. `huggingface/text-generation/` ```python import os @@ -75,7 +245,7 @@ messages = [{ "content": "There's a llama in my garden 😱 What should I do?"," # e.g. Call 'roneneldan/TinyStories-3M' hosted on HF Inference endpoints response = completion( - model="huggingface/roneneldan/TinyStories-3M", + model="huggingface/text-generation/roneneldan/TinyStories-3M", messages=messages, api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", ) diff --git a/docs/my-website/docs/providers/mistral.md b/docs/my-website/docs/providers/mistral.md index 8e5e2bf66..9d13fd017 100644 --- a/docs/my-website/docs/providers/mistral.md +++ b/docs/my-website/docs/providers/mistral.md @@ -44,14 +44,14 @@ for chunk in response: ## Supported Models All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/c1b25538277206b9f00de5254d80d6a83bb19a29/model_prices_and_context_window.json). -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| mistral-tiny | `completion(model="mistral/mistral-tiny", messages)` | -| mistral-small | `completion(model="mistral/mistral-small", messages)` | -| mistral-medium | `completion(model="mistral/mistral-medium", messages)` | -| mistral-large-latest | `completion(model="mistral/mistral-large-latest", messages)` | -| open-mixtral-8x22b | `completion(model="mistral/open-mixtral-8x22b", messages)` | - +| Model Name | Function Call | +|----------------|--------------------------------------------------------------| +| Mistral Small | `completion(model="mistral/mistral-small-latest", messages)` | +| Mistral Medium | `completion(model="mistral/mistral-medium-latest", messages)`| +| Mistral Large | `completion(model="mistral/mistral-large-latest", messages)` | +| Mistral 7B | `completion(model="mistral/open-mistral-7b", messages)` | +| Mixtral 8x7B | `completion(model="mistral/open-mixtral-8x7b", messages)` | +| Mixtral 8x22B | `completion(model="mistral/open-mixtral-8x22b", messages)` | ## Function Calling @@ -116,6 +116,6 @@ All models listed here https://docs.mistral.ai/platform/endpoints are supported | Model Name | Function Call | |--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| mistral-embed | `embedding(model="mistral/mistral-embed", input)` | +| Mistral Embeddings | `embedding(model="mistral/mistral-embed", input)` | diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index a8fe541fc..c44a67412 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -20,7 +20,7 @@ os.environ["OPENAI_API_KEY"] = "your-api-key" # openai call response = completion( - model = "gpt-3.5-turbo", + model = "gpt-4o", messages=[{ "content": "Hello, how are you?","role": "user"}] ) ``` @@ -163,6 +163,8 @@ os.environ["OPENAI_API_BASE"] = "openaiai-api-base" # OPTIONAL | Model Name | Function Call | |-----------------------|-----------------------------------------------------------------| +| gpt-4o | `response = completion(model="gpt-4o", messages=messages)` | +| gpt-4o-2024-05-13 | `response = completion(model="gpt-4o-2024-05-13", messages=messages)` | | gpt-4-turbo | `response = completion(model="gpt-4-turbo", messages=messages)` | | gpt-4-turbo-preview | `response = completion(model="gpt-4-0125-preview", messages=messages)` | | gpt-4-0125-preview | `response = completion(model="gpt-4-0125-preview", messages=messages)` | diff --git a/docs/my-website/docs/providers/predibase.md b/docs/my-website/docs/providers/predibase.md new file mode 100644 index 000000000..3d5bbaef4 --- /dev/null +++ b/docs/my-website/docs/providers/predibase.md @@ -0,0 +1,247 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 🆕 Predibase + +LiteLLM supports all models on Predibase + + +## Usage + + + + +### API KEYS +```python +import os +os.environ["PREDIBASE_API_KEY"] = "" +``` + +### Example Call + +```python +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" +os.environ["PREDIBASE_TENANT_ID"] = "predibase tenant id" + +# predibase llama-3 call +response = completion( + model="predibase/llama-3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + tenant_id: os.environ/PREDIBASE_TENANT_ID + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="llama-3", + messages = [ + { + "role": "system", + "content": "Be a good human!" + }, + { + "role": "user", + "content": "What do you know about earth?" + } + ] + ) + + print(response) + ``` + + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "llama-3", + "messages": [ + { + "role": "system", + "content": "Be a good human!" + }, + { + "role": "user", + "content": "What do you know about earth?" + } + ], + }' + ``` + + + + + + + + + +## Advanced Usage - Prompt Formatting + +LiteLLM has prompt template mappings for all `meta-llama` llama3 instruct models. [**See Code**](https://github.com/BerriAI/litellm/blob/4f46b4c3975cd0f72b8c5acb2cb429d23580c18a/litellm/llms/prompt_templates/factory.py#L1360) + +To apply a custom prompt template: + + + + +```python +import litellm + +import os +os.environ["PREDIBASE_API_KEY"] = "" + +# Create your own custom prompt template +litellm.register_prompt_template( + model="togethercomputer/LLaMA-2-7B-32K", + initial_prompt_value="You are a good assistant" # [OPTIONAL] + roles={ + "system": { + "pre_message": "[INST] <>\n", # [OPTIONAL] + "post_message": "\n<>\n [/INST]\n" # [OPTIONAL] + }, + "user": { + "pre_message": "[INST] ", # [OPTIONAL] + "post_message": " [/INST]" # [OPTIONAL] + }, + "assistant": { + "pre_message": "\n" # [OPTIONAL] + "post_message": "\n" # [OPTIONAL] + } + } + final_prompt_value="Now answer as best you can:" # [OPTIONAL] +) + +def predibase_custom_model(): + model = "predibase/togethercomputer/LLaMA-2-7B-32K" + response = completion(model=model, messages=messages) + print(response['choices'][0]['message']['content']) + return response + +predibase_custom_model() +``` + + + +```yaml +# Model-specific parameters +model_list: + - model_name: mistral-7b # model alias + litellm_params: # actual params for litellm.completion() + model: "predibase/mistralai/Mistral-7B-Instruct-v0.1" + api_key: os.environ/PREDIBASE_API_KEY + initial_prompt_value: "\n" + roles: {"system":{"pre_message":"<|im_start|>system\n", "post_message":"<|im_end|>"}, "assistant":{"pre_message":"<|im_start|>assistant\n","post_message":"<|im_end|>"}, "user":{"pre_message":"<|im_start|>user\n","post_message":"<|im_end|>"}} + final_prompt_value: "\n" + bos_token: "" + eos_token: "" + max_tokens: 4096 +``` + + + + + +## Passing additional params - max_tokens, temperature +See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input) + +```python +# !pip install litellm +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" + +# predibae llama-3 call +response = completion( + model="predibase/llama3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}], + max_tokens=20, + temperature=0.5 +) +``` + +**proxy** + +```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + max_tokens: 20 + temperature: 0.5 +``` + +## Passings Predibase specific params - adapter_id, adapter_source, +Send params [not supported by `litellm.completion()`](https://docs.litellm.ai/docs/completion/input) but supported by Predibase by passing them to `litellm.completion` + +Example `adapter_id`, `adapter_source` are Predibase specific param - [See List](https://github.com/BerriAI/litellm/blob/8a35354dd6dbf4c2fcefcd6e877b980fcbd68c58/litellm/llms/predibase.py#L54) + +```python +# !pip install litellm +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" + +# predibase llama3 call +response = completion( + model="predibase/llama-3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}], + adapter_id="my_repo/3", + adapter_soruce="pbase", +) +``` + +**proxy** + +```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + adapter_id: my_repo/3 + adapter_source: pbase +``` diff --git a/docs/my-website/docs/providers/triton-inference-server.md b/docs/my-website/docs/providers/triton-inference-server.md new file mode 100644 index 000000000..aacc46a39 --- /dev/null +++ b/docs/my-website/docs/providers/triton-inference-server.md @@ -0,0 +1,95 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Triton Inference Server + +LiteLLM supports Embedding Models on Triton Inference Servers + + +## Usage + + + + + +### Example Call + +Use the `triton/` prefix to route to triton server +```python +from litellm import embedding +import os + +response = await litellm.aembedding( + model="triton/", + api_base="https://your-triton-api-base/triton/embeddings", # /embeddings endpoint you want litellm to call on your server + input=["good morning from litellm"], +) +``` + + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: my-triton-model + litellm_params: + model: triton/" + api_base: https://your-triton-api-base/triton/embeddings + ``` + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --detailed_debug + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + from openai import OpenAI + + # set base_url to your proxy server + # set api_key to send to proxy server + client = OpenAI(api_key="", base_url="http://0.0.0.0:4000") + + response = client.embeddings.create( + input=["hello from litellm"], + model="my-triton-model" + ) + + print(response) + + ``` + + + + + + `--header` is optional, only required if you're using litellm proxy with Virtual Keys + + ```shell + curl --location 'http://0.0.0.0:4000/embeddings' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --data ' { + "model": "my-triton-model", + "input": ["write a litellm poem"] + }' + + ``` + + + + + + + + diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 014a29bf1..8d59088af 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -477,6 +477,36 @@ print(response) | code-gecko@latest| `completion('code-gecko@latest', messages)` | +## Embedding Models + +#### Usage - Embedding +```python +import litellm +from litellm import embedding +litellm.vertex_project = "hardy-device-38811" # Your Project ID +litellm.vertex_location = "us-central1" # proj location + +response = embedding( + model="vertex_ai/textembedding-gecko", + input=["good morning from litellm"], +) +print(response) +``` + +#### Supported Embedding Models +All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` | +| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` | +| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` | +| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` | +| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | +| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` | +| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | + + ## Extra ### Using `GOOGLE_APPLICATION_CREDENTIALS` @@ -520,6 +550,12 @@ def load_vertex_ai_credentials(): ### Using GCP Service Account +:::info + +Trying to deploy LiteLLM on Google Cloud Run? Tutorial [here](https://docs.litellm.ai/docs/proxy/deploy#deploy-on-google-cloud-run) + +::: + 1. Figure out the Service Account bound to the Google Cloud Run service diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 4275e0bf0..19f3c88a6 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -1,14 +1,22 @@ # 🚨 Alerting Get alerts for: + - Hanging LLM api calls - Failed LLM api calls - Slow LLM api calls - Budget Tracking per key/user: - When a User/Key crosses their Budget - When a User/Key is 15% away from crossing their Budget +- Spend Reports - Weekly & Monthly spend per Team, Tag - Failed db read/writes +As a bonus, you can also get "daily reports" posted to your slack channel. +These reports contain key metrics like: + +- Top 5 deployments with most failed requests +- Top 5 slowest deployments + ## Quick Start Set up a slack alert channel to receive alerts from proxy. @@ -20,7 +28,8 @@ Get a slack webhook url from https://api.slack.com/messaging/webhooks ### Step 2: Update config.yaml -Let's save a bad key to our proxy +- Set `SLACK_WEBHOOK_URL` in your proxy env to enable Slack alerts. +- Just for testing purposes, let's save a bad key to our proxy. ```yaml model_list: @@ -33,13 +42,11 @@ general_settings: alerting: ["slack"] alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+ +environment_variables: + SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/<>/<>/<>" + SLACK_DAILY_REPORT_FREQUENCY: "86400" # 24 hours; Optional: defaults to 12 hours ``` -Set `SLACK_WEBHOOK_URL` in your proxy env - -```shell -SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/<>/<>/<>" -``` ### Step 3: Start proxy diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index 887ec9e3e..d4760d83f 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -1,8 +1,136 @@ -# Cost Tracking - Azure +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 💸 Spend Tracking + +Track spend for keys, users, and teams across 100+ LLMs. + +## Getting Spend Reports - To Charge Other Teams, API Keys + +Use the `/global/spend/report` endpoint to get daily spend per team, with a breakdown of spend per API Key, Model + +### Example Request + +```shell +curl -X GET 'http://localhost:4000/global/spend/report?start_date=2023-04-01&end_date=2024-06-30' \ + -H 'Authorization: Bearer sk-1234' +``` + +### Example Response + + + + +```shell +[ + { + "group_by_day": "2024-04-30T00:00:00+00:00", + "teams": [ + { + "team_name": "Prod Team", + "total_spend": 0.0015265, + "metadata": [ # see the spend by unique(key + model) + { + "model": "gpt-4", + "spend": 0.00123, + "total_tokens": 28, + "api_key": "88dc28.." # the hashed api key + }, + { + "model": "gpt-4", + "spend": 0.00123, + "total_tokens": 28, + "api_key": "a73dc2.." # the hashed api key + }, + { + "model": "chatgpt-v-2", + "spend": 0.000214, + "total_tokens": 122, + "api_key": "898c28.." # the hashed api key + }, + { + "model": "gpt-3.5-turbo", + "spend": 0.0000825, + "total_tokens": 85, + "api_key": "84dc28.." # the hashed api key + } + ] + } + ] + } +] +``` + + + + + + +```python +import requests +url = 'http://localhost:4000/global/spend/report' +params = { + 'start_date': '2023-04-01', + 'end_date': '2024-06-30' +} + +headers = { + 'Authorization': 'Bearer sk-1234' +} + +# Make the GET request +response = requests.get(url, headers=headers, params=params) +spend_report = response.json() + +for row in spend_report: + date = row["group_by_day"] + teams = row["teams"] + for team in teams: + team_name = team["team_name"] + total_spend = team["total_spend"] + metadata = team["metadata"] + + print(f"Date: {date}") + print(f"Team: {team_name}") + print(f"Total Spend: {total_spend}") + print("Metadata: ", metadata) + print() +``` + +Output from script +```shell +# Date: 2024-05-11T00:00:00+00:00 +# Team: local_test_team +# Total Spend: 0.003675099999999999 +# Metadata: [{'model': 'gpt-3.5-turbo', 'spend': 0.003675099999999999, 'api_key': 'b94d5e0bc3a71a573917fe1335dc0c14728c7016337451af9714924ff3a729db', 'total_tokens': 3105}] + +# Date: 2024-05-13T00:00:00+00:00 +# Team: Unassigned Team +# Total Spend: 3.4e-05 +# Metadata: [{'model': 'gpt-3.5-turbo', 'spend': 3.4e-05, 'api_key': '9569d13c9777dba68096dea49b0b03e0aaf4d2b65d4030eda9e8a2733c3cd6e0', 'total_tokens': 50}] + +# Date: 2024-05-13T00:00:00+00:00 +# Team: central +# Total Spend: 0.000684 +# Metadata: [{'model': 'gpt-3.5-turbo', 'spend': 0.000684, 'api_key': '0323facdf3af551594017b9ef162434a9b9a8ca1bbd9ccbd9d6ce173b1015605', 'total_tokens': 498}] + +# Date: 2024-05-13T00:00:00+00:00 +# Team: local_test_team +# Total Spend: 0.0005715000000000001 +# Metadata: [{'model': 'gpt-3.5-turbo', 'spend': 0.0005715000000000001, 'api_key': 'b94d5e0bc3a71a573917fe1335dc0c14728c7016337451af9714924ff3a729db', 'total_tokens': 423}] +``` + + + + + + + +## Spend Tracking for Azure Set base model for cost tracking azure image-gen call -## Image Generation +### Image Generation ```yaml model_list: @@ -17,7 +145,7 @@ model_list: mode: image_generation ``` -## Chat Completions / Embeddings +### Chat Completions / Embeddings **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking diff --git a/docs/my-website/docs/proxy/customer_routing.md b/docs/my-website/docs/proxy/customer_routing.md new file mode 100644 index 000000000..4c8a60af8 --- /dev/null +++ b/docs/my-website/docs/proxy/customer_routing.md @@ -0,0 +1,83 @@ +# Region-based Routing + +Route specific customers to eu-only models. + +By specifying 'allowed_model_region' for a customer, LiteLLM will filter-out any models in a model group which is not in the allowed region (i.e. 'eu'). + +[**See Code**](https://github.com/BerriAI/litellm/blob/5eb12e30cc5faa73799ebc7e48fc86ebf449c879/litellm/router.py#L2938) + +### 1. Create customer with region-specification + +Use the litellm 'end-user' object for this. + +End-users can be tracked / id'ed by passing the 'user' param to litellm in an openai chat completion/embedding call. + +```bash +curl -X POST --location 'http://0.0.0.0:4000/end_user/new' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "user_id" : "ishaan-jaff-45", + "allowed_model_region": "eu", # 👈 SPECIFY ALLOWED REGION='eu' +}' +``` + +### 2. Add eu models to model-group + +Add eu models to a model group. For azure models, litellm can automatically infer the region (no need to set it). + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/gpt-35-turbo-eu # 👈 EU azure model + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/chatgpt-v-2 + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + api_version: "2023-05-15" + api_key: os.environ/AZURE_API_KEY + +router_settings: + enable_pre_call_checks: true # 👈 IMPORTANT +``` + +Start the proxy + +```yaml +litellm --config /path/to/config.yaml +``` + +### 3. Test it! + +Make a simple chat completions call to the proxy. In the response headers, you should see the returned api base. + +```bash +curl -X POST --location 'http://localhost:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what is the meaning of the universe? 1234" + }], + "user": "ishaan-jaff-45" # 👈 USER ID +} +' +``` + +Expected API Base in response headers + +``` +x-litellm-api-base: "https://my-endpoint-europe-berri-992.openai.azure.com/" +``` + +### FAQ + +**What happens if there are no available models for that region?** + +Since the router filters out models not in the specified region, it will return back as an error to the user, if no models in that region are available. \ No newline at end of file diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 1c3b4f81c..538a81d4b 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -3,7 +3,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🔎 Logging - Custom Callbacks, DataDog, Langfuse, s3 Bucket, Sentry, OpenTelemetry, Athina +# 🔎 Logging - Custom Callbacks, DataDog, Langfuse, s3 Bucket, Sentry, OpenTelemetry, Athina, Azure Content-Safety Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTelemetry, LangFuse, DynamoDB, s3 Bucket @@ -17,6 +17,7 @@ Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTeleme - [Logging to Sentry](#logging-proxy-inputoutput---sentry) - [Logging to Traceloop (OpenTelemetry)](#logging-proxy-inputoutput-traceloop-opentelemetry) - [Logging to Athina](#logging-proxy-inputoutput-athina) +- [(BETA) Moderation with Azure Content-Safety](#moderation-with-azure-content-safety) ## Custom Callback Class [Async] Use this when you want to run custom callbacks in `python` @@ -914,39 +915,72 @@ Test Request litellm --test ``` -## Logging Proxy Input/Output Traceloop (OpenTelemetry) +## Logging Proxy Input/Output in OpenTelemetry format using Traceloop's OpenLLMetry -Traceloop allows you to log LLM Input/Output in the OpenTelemetry format +[OpenLLMetry](https://github.com/traceloop/openllmetry) _(built and maintained by Traceloop)_ is a set of extensions +built on top of [OpenTelemetry](https://opentelemetry.io/) that gives you complete observability over your LLM +application. Because it uses OpenTelemetry under the +hood, [it can be connected to various observability solutions](https://www.traceloop.com/docs/openllmetry/integrations/introduction) +like: -We will use the `--config` to set `litellm.success_callback = ["traceloop"]` this will log all successfull LLM calls to traceloop +* [Traceloop](https://www.traceloop.com/docs/openllmetry/integrations/traceloop) +* [Axiom](https://www.traceloop.com/docs/openllmetry/integrations/axiom) +* [Azure Application Insights](https://www.traceloop.com/docs/openllmetry/integrations/azure) +* [Datadog](https://www.traceloop.com/docs/openllmetry/integrations/datadog) +* [Dynatrace](https://www.traceloop.com/docs/openllmetry/integrations/dynatrace) +* [Grafana Tempo](https://www.traceloop.com/docs/openllmetry/integrations/grafana) +* [Honeycomb](https://www.traceloop.com/docs/openllmetry/integrations/honeycomb) +* [HyperDX](https://www.traceloop.com/docs/openllmetry/integrations/hyperdx) +* [Instana](https://www.traceloop.com/docs/openllmetry/integrations/instana) +* [New Relic](https://www.traceloop.com/docs/openllmetry/integrations/newrelic) +* [OpenTelemetry Collector](https://www.traceloop.com/docs/openllmetry/integrations/otel-collector) +* [Service Now Cloud Observability](https://www.traceloop.com/docs/openllmetry/integrations/service-now) +* [Sentry](https://www.traceloop.com/docs/openllmetry/integrations/sentry) +* [SigNoz](https://www.traceloop.com/docs/openllmetry/integrations/signoz) +* [Splunk](https://www.traceloop.com/docs/openllmetry/integrations/splunk) -**Step 1** Install traceloop-sdk and set Traceloop API key +We will use the `--config` to set `litellm.success_callback = ["traceloop"]` to achieve this, steps are listed below. + +**Step 1:** Install the SDK ```shell -pip install traceloop-sdk -U +pip install traceloop-sdk ``` -Traceloop outputs standard OpenTelemetry data that can be connected to your observability stack. Send standard OpenTelemetry from LiteLLM Proxy to [Traceloop](https://www.traceloop.com/docs/openllmetry/integrations/traceloop), [Dynatrace](https://www.traceloop.com/docs/openllmetry/integrations/dynatrace), [Datadog](https://www.traceloop.com/docs/openllmetry/integrations/datadog) -, [New Relic](https://www.traceloop.com/docs/openllmetry/integrations/newrelic), [Honeycomb](https://www.traceloop.com/docs/openllmetry/integrations/honeycomb), [Grafana Tempo](https://www.traceloop.com/docs/openllmetry/integrations/grafana), [Splunk](https://www.traceloop.com/docs/openllmetry/integrations/splunk), [OpenTelemetry Collector](https://www.traceloop.com/docs/openllmetry/integrations/otel-collector) +**Step 2:** Configure Environment Variable for trace exporting + +You will need to configure where to export your traces. Environment variables will control this, example: For Traceloop +you should use `TRACELOOP_API_KEY`, whereas for Datadog you use `TRACELOOP_BASE_URL`. For more +visit [the Integrations Catalog](https://www.traceloop.com/docs/openllmetry/integrations/introduction). + +If you are using Datadog as the observability solutions then you can set `TRACELOOP_BASE_URL` as: + +```shell +TRACELOOP_BASE_URL=http://:4318 +``` + +**Step 3**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` -**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` ```yaml model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-3.5-turbo litellm_params: model: gpt-3.5-turbo + api_key: my-fake-key # replace api_key with actual key litellm_settings: - success_callback: ["traceloop"] + success_callback: [ "traceloop" ] ``` -**Step 3**: Start the proxy, make a test request +**Step 4**: Start the proxy, make a test request Start proxy + ```shell litellm --config config.yaml --debug ``` Test Request + ``` curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ @@ -1003,4 +1037,87 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ] }' -``` \ No newline at end of file +``` + +## (BETA) Moderation with Azure Content Safety + +[Azure Content-Safety](https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety) is a Microsoft Azure service that provides content moderation APIs to detect potential offensive, harmful, or risky content in text. + +We will use the `--config` to set `litellm.success_callback = ["azure_content_safety"]` this will moderate all LLM calls using Azure Content Safety. + +**Step 0** Deploy Azure Content Safety + +Deploy an Azure Content-Safety instance from the Azure Portal and get the `endpoint` and `key`. + +**Step 1** Set Athina API key + +```shell +AZURE_CONTENT_SAFETY_KEY = "" +``` + +**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + callbacks: ["azure_content_safety"] + azure_content_safety_params: + endpoint: "" + key: "os.environ/AZURE_CONTENT_SAFETY_KEY" +``` + +**Step 3**: Start the proxy, make a test request + +Start proxy +```shell +litellm --config config.yaml --debug +``` + +Test Request +``` +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Hi, how are you?" + } + ] + }' +``` + +An HTTP 400 error will be returned if the content is detected with a value greater than the threshold set in the `config.yaml`. +The details of the response will describe : +- The `source` : input text or llm generated text +- The `category` : the category of the content that triggered the moderation +- The `severity` : the severity from 0 to 10 + +**Step 4**: Customizing Azure Content Safety Thresholds + +You can customize the thresholds for each category by setting the `thresholds` in the `config.yaml` + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + callbacks: ["azure_content_safety"] + azure_content_safety_params: + endpoint: "" + key: "os.environ/AZURE_CONTENT_SAFETY_KEY" + thresholds: + Hate: 6 + SelfHarm: 8 + Sexual: 6 + Violence: 4 +``` + +:::info +`thresholds` are not required by default, but you can tune the values to your needs. +Default values is `4` for all categories +::: \ No newline at end of file diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 980bba542..32cd916c9 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -3,34 +3,38 @@ import TabItem from '@theme/TabItem'; # ⚡ Best Practices for Production -Expected Performance in Production +## 1. Use this config.yaml +Use this config.yaml in production (with your own LLMs) -1 LiteLLM Uvicorn Worker on Kubernetes - -| Description | Value | -|--------------|-------| -| Avg latency | `50ms` | -| Median latency | `51ms` | -| `/chat/completions` Requests/second | `35` | -| `/chat/completions` Requests/minute | `2100` | -| `/chat/completions` Requests/hour | `126K` | - - -## 1. Switch off Debug Logging - -Remove `set_verbose: True` from your config.yaml ```yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +general_settings: + master_key: sk-1234 # enter your own master key, ensure it starts with 'sk-' + alerting: ["slack"] # Setup slack alerting - get alerts on LLM exceptions, Budget Alerts, Slow LLM Responses + proxy_batch_write_at: 60 # Batch write spend updates every 60s + litellm_settings: - set_verbose: True + set_verbose: False # Switch off Debug Logging, ensure your logs do not have any debugging on ``` -You should only see the following level of details in logs on the proxy server +Set slack webhook url in your env ```shell -# INFO: 192.168.2.205:11774 - "POST /chat/completions HTTP/1.1" 200 OK -# INFO: 192.168.2.205:34717 - "POST /chat/completions HTTP/1.1" 200 OK -# INFO: 192.168.2.205:29734 - "POST /chat/completions HTTP/1.1" 200 OK +export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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" ``` +:::info + +Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + + ## 2. On Kubernetes - Use 1 Uvicorn worker [Suggested CMD] Use this Docker `CMD`. This will start the proxy with 1 Uvicorn Async Worker @@ -40,21 +44,12 @@ Use this Docker `CMD`. This will start the proxy with 1 Uvicorn Async Worker CMD ["--port", "4000", "--config", "./proxy_server_config.yaml"] ``` -## 3. Batch write spend updates every 60s -The default proxy batch write is 10s. This is to make it easy to see spend when debugging locally. +## 3. Use Redis 'port','host', 'password'. NOT 'redis_url' -In production, we recommend using a longer interval period of 60s. This reduces the number of connections used to make DB writes. +If you decide to use Redis, DO NOT use 'redis_url'. We recommend usig redis port, host, and password params. -```yaml -general_settings: - master_key: sk-1234 - proxy_batch_write_at: 60 # 👈 Frequency of batch writing logs to server (in seconds) -``` - -## 4. use Redis 'port','host', 'password'. NOT 'redis_url' - -When connecting to Redis use redis port, host, and password params. Not 'redis_url'. We've seen a 80 RPS difference between these 2 approaches when using the async redis client. +`redis_url`is 80 RPS slower This is still something we're investigating. Keep track of it [here](https://github.com/BerriAI/litellm/issues/3188) @@ -69,103 +64,31 @@ router_settings: redis_password: os.environ/REDIS_PASSWORD ``` -## 5. Switch off resetting budgets +## Extras +### Expected Performance in Production -Add this to your config.yaml. (Only spend per Key, User and Team will be tracked - spend per API Call will not be written to the LiteLLM Database) -```yaml -general_settings: - disable_reset_budget: true -``` +1 LiteLLM Uvicorn Worker on Kubernetes -## 6. Move spend logs to separate server (BETA) - -Writing each spend log to the db can slow down your proxy. In testing we saw a 70% improvement in median response time, by moving writing spend logs to a separate server. - -👉 [LiteLLM Spend Logs Server](https://github.com/BerriAI/litellm/tree/main/litellm-js/spend-logs) +| Description | Value | +|--------------|-------| +| Avg latency | `50ms` | +| Median latency | `51ms` | +| `/chat/completions` Requests/second | `35` | +| `/chat/completions` Requests/minute | `2100` | +| `/chat/completions` Requests/hour | `126K` | -**Spend Logs** -This is a log of the key, tokens, model, and latency for each call on the proxy. +### Verifying Debugging logs are off -[**Full Payload**](https://github.com/BerriAI/litellm/blob/8c9623a6bc4ad9da0a2dac64249a60ed8da719e8/litellm/proxy/utils.py#L1769) - - -**1. Start the spend logs server** - -```bash -docker run -p 3000:3000 \ - -e DATABASE_URL="postgres://.." \ - ghcr.io/berriai/litellm-spend_logs:main-latest - -# RUNNING on http://0.0.0.0:3000 -``` - -**2. Connect to proxy** - - -Example litellm_config.yaml - -```yaml -model_list: -- model_name: fake-openai-endpoint - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -general_settings: - master_key: sk-1234 - proxy_batch_write_at: 5 # 👈 Frequency of batch writing logs to server (in seconds) -``` - -Add `SPEND_LOGS_URL` as an environment variable when starting the proxy - -```bash -docker run \ - -v $(pwd)/litellm_config.yaml:/app/config.yaml \ - -e DATABASE_URL="postgresql://.." \ - -e SPEND_LOGS_URL="http://host.docker.internal:3000" \ # 👈 KEY CHANGE - -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ - --config /app/config.yaml --detailed_debug - -# Running on http://0.0.0.0:4000 -``` - -**3. Test Proxy!** - - -```bash -curl --location 'http://0.0.0.0:4000/v1/chat/completions' \ ---header 'Content-Type: application/json' \ ---header 'Authorization: Bearer sk-1234' \ ---data '{ - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "Be helpful"}, - {"role": "user", "content": "What do you know?"} - ] -}' -``` - -In your LiteLLM Spend Logs Server, you should see - -**Expected Response** - -``` -Received and stored 1 logs. Total logs in memory: 1 -... -Flushed 1 log to the DB. +You should only see the following level of details in logs on the proxy server +```shell +# INFO: 192.168.2.205:11774 - "POST /chat/completions HTTP/1.1" 200 OK +# INFO: 192.168.2.205:34717 - "POST /chat/completions HTTP/1.1" 200 OK +# INFO: 192.168.2.205:29734 - "POST /chat/completions HTTP/1.1" 200 OK ``` -### Machine Specification - -A t2.micro should be sufficient to handle 1k logs / minute on this server. - -This consumes at max 120MB, and <0.1 vCPU. - -## Machine Specifications to Deploy LiteLLM +### Machine Specifications to Deploy LiteLLM | Service | Spec | CPUs | Memory | Architecture | Version| | --- | --- | --- | --- | --- | --- | @@ -173,7 +96,7 @@ This consumes at max 120MB, and <0.1 vCPU. | Redis Cache | - | - | - | - | 7.0+ Redis Engine| -## Reference Kubernetes Deployment YAML +### Reference Kubernetes Deployment YAML Reference Kubernetes `deployment.yaml` that was load tested by us diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index bd04216dd..e39a6765f 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -151,7 +151,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` -## Advanced - Context Window Fallbacks +## Advanced - Context Window Fallbacks (Pre-Call Checks + Fallbacks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -232,16 +232,16 @@ model_list: - model_name: gpt-3.5-turbo-small litellm_params: model: azure/chatgpt-v-2 - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-07-01-preview" - model_info: - base_model: azure/gpt-4-1106-preview # 2. 👈 (azure-only) SET BASE MODEL + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2023-07-01-preview" + model_info: + base_model: azure/gpt-4-1106-preview # 2. 👈 (azure-only) SET BASE MODEL - model_name: gpt-3.5-turbo-large litellm_params: - model: gpt-3.5-turbo-1106 - api_key: os.environ/OPENAI_API_KEY + model: gpt-3.5-turbo-1106 + api_key: os.environ/OPENAI_API_KEY - model_name: claude-opus litellm_params: @@ -287,6 +287,69 @@ print(response) +## Advanced - EU-Region Filtering (Pre-Call Checks) + +**Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. + +Set 'region_name' of deployment. + +**Note:** LiteLLM can automatically infer region_name for Vertex AI, Bedrock, and IBM WatsonxAI based on your litellm params. For Azure, set `litellm.enable_preview = True`. + +**1. Set Config** + +```yaml +router_settings: + enable_pre_call_checks: true # 1. Enable pre-call checks + +model_list: +- model_name: gpt-3.5-turbo + litellm_params: + model: azure/chatgpt-v-2 + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2023-07-01-preview" + region_name: "eu" # 👈 SET EU-REGION + +- model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo-1106 + api_key: os.environ/OPENAI_API_KEY + +- model_name: gemini-pro + litellm_params: + model: vertex_ai/gemini-pro-1.5 + vertex_project: adroit-crow-1234 + vertex_location: us-east1 # 👈 AUTOMATICALLY INFERS 'region_name' +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +**3. Test it!** + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.with_raw_response.create( + model="gpt-3.5-turbo", + messages = [{"role": "user", "content": "Who was Alexander?"}] +) + +print(response) + +print(f"response.headers.get('x-litellm-model-api-base')") +``` + ## Advanced - Custom Timeouts, Stream Timeouts - Per Model For each model you can set `timeout` & `stream_timeout` under `litellm_params` ```yaml diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index 81475951f..659cc6edf 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -17,6 +17,7 @@ This is a new feature, and subject to changes based on feedback. ### Step 1. Setup Proxy - `JWT_PUBLIC_KEY_URL`: This is the public keys endpoint of your OpenID provider. Typically it's `{openid-provider-base-url}/.well-known/openid-configuration/jwks`. For Keycloak it's `{keycloak_base_url}/realms/{your-realm}/protocol/openid-connect/certs`. +- `JWT_AUDIENCE`: This is the audience used for decoding the JWT. If not set, the decode step will not verify the audience. ```bash export JWT_PUBLIC_KEY_URL="" # "https://demo.duendesoftware.com/.well-known/openid-configuration/jwks" @@ -109,7 +110,7 @@ general_settings: admin_jwt_scope: "litellm-proxy-admin" ``` -## Advanced - Spend Tracking (User / Team / Org) +## Advanced - Spend Tracking (End-Users / Internal Users / Team / Org) Set the field in the jwt token, which corresponds to a litellm user / team / org. @@ -122,6 +123,7 @@ general_settings: team_id_jwt_field: "client_id" # 👈 CAN BE ANY FIELD user_id_jwt_field: "sub" # 👈 CAN BE ANY FIELD org_id_jwt_field: "org_id" # 👈 CAN BE ANY FIELD + end_user_id_jwt_field: "customer_id" # 👈 CAN BE ANY FIELD ``` Expected JWT: @@ -130,7 +132,7 @@ Expected JWT: { "client_id": "my-unique-team", "sub": "my-unique-user", - "org_id": "my-unique-org" + "org_id": "my-unique-org", } ``` diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index fa78b37c1..cda3a46af 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -365,6 +365,188 @@ curl --location 'http://0.0.0.0:4000/moderations' \ ## Advanced +### (BETA) Batch Completions - pass multiple models + +Use this when you want to send 1 request to N Models + +#### Expected Request Format + +Pass model as a string of comma separated value of models. Example `"model"="llama3,gpt-3.5-turbo"` + +This same request will be sent to the following model groups on the [litellm proxy config.yaml](https://docs.litellm.ai/docs/proxy/configs) +- `model_name="llama3"` +- `model_name="gpt-3.5-turbo"` + + + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.chat.completions.create( + model="gpt-3.5-turbo,llama3", + messages=[ + {"role": "user", "content": "this is a test request, write a short poem"} + ], +) + +print(response) +``` + + + +#### Expected Response Format + +Get a list of responses when `model` is passed as a list + +```python +[ + ChatCompletion( + id='chatcmpl-9NoYhS2G0fswot0b6QpoQgmRQMaIf', + choices=[ + Choice( + finish_reason='stop', + index=0, + logprobs=None, + message=ChatCompletionMessage( + content='In the depths of my soul, a spark ignites\nA light that shines so pure and bright\nIt dances and leaps, refusing to die\nA flame of hope that reaches the sky\n\nIt warms my heart and fills me with bliss\nA reminder that in darkness, there is light to kiss\nSo I hold onto this fire, this guiding light\nAnd let it lead me through the darkest night.', + role='assistant', + function_call=None, + tool_calls=None + ) + ) + ], + created=1715462919, + model='gpt-3.5-turbo-0125', + object='chat.completion', + system_fingerprint=None, + usage=CompletionUsage( + completion_tokens=83, + prompt_tokens=17, + total_tokens=100 + ) + ), + ChatCompletion( + id='chatcmpl-4ac3e982-da4e-486d-bddb-ed1d5cb9c03c', + choices=[ + Choice( + finish_reason='stop', + index=0, + logprobs=None, + message=ChatCompletionMessage( + content="A test request, and I'm delighted!\nHere's a short poem, just for you:\n\nMoonbeams dance upon the sea,\nA path of light, for you to see.\nThe stars up high, a twinkling show,\nA night of wonder, for all to know.\n\nThe world is quiet, save the night,\nA peaceful hush, a gentle light.\nThe world is full, of beauty rare,\nA treasure trove, beyond compare.\n\nI hope you enjoyed this little test,\nA poem born, of whimsy and jest.\nLet me know, if there's anything else!", + role='assistant', + function_call=None, + tool_calls=None + ) + ) + ], + created=1715462919, + model='groq/llama3-8b-8192', + object='chat.completion', + system_fingerprint='fp_a2c8d063cb', + usage=CompletionUsage( + completion_tokens=120, + prompt_tokens=20, + total_tokens=140 + ) + ) +] +``` + + + + + + + + + +```shell +curl --location 'http://localhost:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "llama3,gpt-3.5-turbo", + "max_tokens": 10, + "user": "litellm2", + "messages": [ + { + "role": "user", + "content": "is litellm getting better" + } + ] +}' +``` + + + + +#### Expected Response Format + +Get a list of responses when `model` is passed as a list + +```json +[ + { + "id": "chatcmpl-3dbd5dd8-7c82-4ca3-bf1f-7c26f497cf2b", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "The Elder Scrolls IV: Oblivion!\n\nReleased", + "role": "assistant" + } + } + ], + "created": 1715459876, + "model": "groq/llama3-8b-8192", + "object": "chat.completion", + "system_fingerprint": "fp_179b0f92c9", + "usage": { + "completion_tokens": 10, + "prompt_tokens": 12, + "total_tokens": 22 + } + }, + { + "id": "chatcmpl-9NnldUfFLmVquFHSX4yAtjCw8PGei", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "TES4 could refer to The Elder Scrolls IV:", + "role": "assistant" + } + } + ], + "created": 1715459877, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 10, + "prompt_tokens": 9, + "total_tokens": 19 + } + } +] +``` + + + + + + + + + ### Pass User LLM API Keys, Fallbacks Allow your end-users to pass their model list, api base, OpenAI API key (any LiteLLM supported provider) to make requests diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 478d63f84..6d9c43c5f 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -12,8 +12,8 @@ Requirements: You can set budgets at 3 levels: - For the proxy -- For a user -- For a 'user' passed to `/chat/completions`, `/embeddings` etc +- For an internal user +- For an end-user - For a key - For a key (model specific budgets) @@ -58,7 +58,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` - + Apply a budget across multiple keys. @@ -165,12 +165,12 @@ curl --location 'http://localhost:4000/team/new' \ } ``` - + Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user** **Step 1. Modify config.yaml** -Define `litellm.max_user_budget` +Define `litellm.max_end_user_budget` ```yaml general_settings: master_key: sk-1234 @@ -328,7 +328,7 @@ You can set: - max parallel requests - + Use `/user/new`, to persist rate limits across multiple keys. @@ -408,7 +408,7 @@ curl --location 'http://localhost:4000/user/new' \ ``` -## Create new keys for existing user +## Create new keys for existing internal user Just include user_id in the `/key/generate` request. diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 0aa7901c0..5ba3221c9 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -96,7 +96,7 @@ print(response) - `router.aimage_generation()` - async image generation calls ## Advanced - Routing Strategies -#### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based +#### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based, Cost Based Router provides 4 strategies for routing your calls across multiple deployments: @@ -467,6 +467,101 @@ async def router_acompletion(): asyncio.run(router_acompletion()) ``` + + + +Picks a deployment based on the lowest cost + +How this works: +- Get all healthy deployments +- Select all deployments that are under their provided `rpm/tpm` limits +- For each deployment check if `litellm_param["model"]` exists in [`litellm_model_cost_map`](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) + - if deployment does not exist in `litellm_model_cost_map` -> use deployment_cost= `$1` +- Select deployment with lowest cost + +```python +from litellm import Router +import asyncio + +model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "openai-gpt-4"}, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "groq/llama3-8b-8192"}, + "model_info": {"id": "groq-llama"}, + }, +] + +# init router +router = Router(model_list=model_list, routing_strategy="cost-based-routing") +async def router_acompletion(): + response = await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}] + ) + print(response) + + print(response._hidden_params["model_id"]) # expect groq-llama, since groq/llama has lowest cost + return response + +asyncio.run(router_acompletion()) + +``` + + +#### Using Custom Input/Output pricing + +Set `litellm_params["input_cost_per_token"]` and `litellm_params["output_cost_per_token"]` for using custom pricing when routing + +```python +model_list = [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/chatgpt-v-2", + "input_cost_per_token": 0.00003, + "output_cost_per_token": 0.00003, + }, + "model_info": {"id": "chatgpt-v-experimental"}, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/chatgpt-v-1", + "input_cost_per_token": 0.000000001, + "output_cost_per_token": 0.00000001, + }, + "model_info": {"id": "chatgpt-v-1"}, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "azure/chatgpt-v-5", + "input_cost_per_token": 10, + "output_cost_per_token": 12, + }, + "model_info": {"id": "chatgpt-v-5"}, + }, +] +# init router +router = Router(model_list=model_list, routing_strategy="cost-based-routing") +async def router_acompletion(): + response = await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}] + ) + print(response) + + print(response._hidden_params["model_id"]) # expect chatgpt-v-1, since chatgpt-v-1 has lowest cost + return response + +asyncio.run(router_acompletion()) +``` + @@ -558,7 +653,9 @@ from litellm import Router model_list = [{...}] router = Router(model_list=model_list, - allowed_fails=1) # cooldown model if it fails > 1 call in a minute. + allowed_fails=1, # cooldown model if it fails > 1 call in a minute. + cooldown_time=100 # cooldown the deployment for 100 seconds if it num_fails > allowed_fails + ) user_message = "Hello, whats the weather in San Francisco??" messages = [{"content": user_message, "role": "user"}] @@ -616,6 +713,57 @@ response = router.completion(model="gpt-3.5-turbo", messages=messages) print(f"response: {response}") ``` +#### Retries based on Error Type + +Use `RetryPolicy` if you want to set a `num_retries` based on the Exception receieved + +Example: +- 4 retries for `ContentPolicyViolationError` +- 0 retries for `RateLimitErrors` + +Example Usage + +```python +from litellm.router import RetryPolicy +retry_policy = RetryPolicy( + ContentPolicyViolationErrorRetries=3, # run 3 retries for ContentPolicyViolationErrors + AuthenticationErrorRetries=0, # run 0 retries for AuthenticationErrorRetries + BadRequestErrorRetries=1, + TimeoutErrorRetries=2, + RateLimitErrorRetries=3, +) + +router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + }, + }, + { + "model_name": "bad-model", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": "bad-key", + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + }, + }, + ], + retry_policy=retry_policy, +) + +response = await router.acompletion( + model=model, + messages=messages, +) +``` + + ### Fallbacks If a call fails after num_retries, fall back to another model group. @@ -624,6 +772,8 @@ If the error is a context window exceeded error, fall back to a larger model gro Fallbacks are done in-order - ["gpt-3.5-turbo, "gpt-4", "gpt-4-32k"], will do 'gpt-3.5-turbo' first, then 'gpt-4', etc. +You can also set 'default_fallbacks', in case a specific model group is misconfigured / bad. + ```python from litellm import Router @@ -684,6 +834,7 @@ model_list = [ router = Router(model_list=model_list, fallbacks=[{"azure/gpt-3.5-turbo": ["gpt-3.5-turbo"]}], + default_fallbacks=["gpt-3.5-turbo-16k"], context_window_fallbacks=[{"azure/gpt-3.5-turbo-context-fallback": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}], set_verbose=True) @@ -733,13 +884,11 @@ router = Router(model_list: Optional[list] = None, cache_responses=True) ``` -## Pre-Call Checks (Context Window) +## Pre-Call Checks (Context Window, EU-Regions) Enable pre-call checks to filter out: 1. deployments with context window limit < messages for a call. -2. deployments that have exceeded rate limits when making concurrent calls. (eg. `asyncio.gather(*[ - router.acompletion(model="gpt-3.5-turbo", messages=m) for m in list_of_messages - ])`) +2. deployments outside of eu-region @@ -754,10 +903,14 @@ router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Set t **2. Set Model List** -For azure deployments, set the base model. Pick the base model from [this list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), all the azure models start with `azure/`. +For context window checks on azure deployments, set the base model. Pick the base model from [this list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), all the azure models start with `azure/`. - - +For 'eu-region' filtering, Set 'region_name' of deployment. + +**Note:** We automatically infer region_name for Vertex AI, Bedrock, and IBM WatsonxAI based on your litellm params. For Azure, set `litellm.enable_preview = True`. + + +[**See Code**](https://github.com/BerriAI/litellm/blob/d33e49411d6503cb634f9652873160cd534dec96/litellm/router.py#L2958) ```python model_list = [ @@ -768,10 +921,9 @@ model_list = [ "api_key": os.getenv("AZURE_API_KEY"), "api_version": os.getenv("AZURE_API_VERSION"), "api_base": os.getenv("AZURE_API_BASE"), - }, - "model_info": { + "region_name": "eu" # 👈 SET 'EU' REGION NAME "base_model": "azure/gpt-35-turbo", # 👈 (Azure-only) SET BASE MODEL - } + }, }, { "model_name": "gpt-3.5-turbo", # model group name @@ -780,54 +932,26 @@ model_list = [ "api_key": os.getenv("OPENAI_API_KEY"), }, }, + { + "model_name": "gemini-pro", + "litellm_params: { + "model": "vertex_ai/gemini-pro-1.5", + "vertex_project": "adroit-crow-1234", + "vertex_location": "us-east1" # 👈 AUTOMATICALLY INFERS 'region_name' + } + } ] router = Router(model_list=model_list, enable_pre_call_checks=True) ``` - - - - -```python -model_list = [ - { - "model_name": "gpt-3.5-turbo-small", # model group name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "model_info": { - "base_model": "azure/gpt-35-turbo", # 👈 (Azure-only) SET BASE MODEL - } - }, - { - "model_name": "gpt-3.5-turbo-large", # model group name - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo-1106", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "claude-opus", - "litellm_params": { call - "model": "claude-3-opus-20240229", - "api_key": os.getenv("ANTHROPIC_API_KEY"), - }, - }, - ] - -router = Router(model_list=model_list, enable_pre_call_checks=True, context_window_fallbacks=[{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}]) -``` - - - - **3. Test it!** + + + + ```python """ - Give a gpt-3.5-turbo model group with different context windows (4k vs. 16k) @@ -837,7 +961,6 @@ router = Router(model_list=model_list, enable_pre_call_checks=True, context_wind from litellm import Router import os -try: model_list = [ { "model_name": "gpt-3.5-turbo", # model group name @@ -846,6 +969,7 @@ model_list = [ "api_key": os.getenv("AZURE_API_KEY"), "api_version": os.getenv("AZURE_API_VERSION"), "api_base": os.getenv("AZURE_API_BASE"), + "base_model": "azure/gpt-35-turbo", }, "model_info": { "base_model": "azure/gpt-35-turbo", @@ -875,6 +999,59 @@ response = router.completion( print(f"response: {response}") ``` + + +```python +""" +- Give 2 gpt-3.5-turbo deployments, in eu + non-eu regions +- Make a call +- Assert it picks the eu-region model +""" + +from litellm import Router +import os + +model_list = [ + { + "model_name": "gpt-3.5-turbo", # model group name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + "region_name": "eu" + }, + "model_info": { + "id": "1" + } + }, + { + "model_name": "gpt-3.5-turbo", # model group name + "litellm_params": { # params for litellm completion/embedding call + "model": "gpt-3.5-turbo-1106", + "api_key": os.getenv("OPENAI_API_KEY"), + }, + "model_info": { + "id": "2" + } + }, +] + +router = Router(model_list=model_list, enable_pre_call_checks=True) + +response = router.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Who was Alexander?"}], +) + +print(f"response: {response}") + +print(f"response id: {response._hidden_params['model_id']}") +``` + + + + :::info @@ -940,6 +1117,46 @@ async def test_acompletion_caching_on_router_caching_groups(): asyncio.run(test_acompletion_caching_on_router_caching_groups()) ``` +## Alerting 🚨 + +Send alerts to slack / your webhook url for the following events +- LLM API Exceptions +- Slow LLM Responses + +Get a slack webhook url from https://api.slack.com/messaging/webhooks + +#### Usage +Initialize an `AlertingConfig` and pass it to `litellm.Router`. The following code will trigger an alert because `api_key=bad-key` which is invalid + +```python +from litellm.router import AlertingConfig +import litellm +import os + +router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "bad_key", + }, + } + ], + alerting_config= AlertingConfig( + alerting_threshold=10, # threshold for slow / hanging llm responses (in seconds). Defaults to 300 seconds + webhook_url= os.getenv("SLACK_WEBHOOK_URL") # webhook you want to send alerts to + ), +) +try: + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) +except: + pass +``` + ## Track cost for Azure Deployments **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking @@ -1097,10 +1314,11 @@ def __init__( num_retries: int = 0, timeout: Optional[float] = None, default_litellm_params={}, # default params for Router.chat.completion.create - fallbacks: List = [], + fallbacks: Optional[List] = None, + default_fallbacks: Optional[List] = None allowed_fails: Optional[int] = None, # Number of times a deployment can failbefore being added to cooldown cooldown_time: float = 1, # (seconds) time to cooldown a deployment after failure - context_window_fallbacks: List = [], + context_window_fallbacks: Optional[List] = None, model_group_alias: Optional[dict] = {}, retry_after: int = 0, # (min) time to wait before retrying a failed request routing_strategy: Literal[ @@ -1108,6 +1326,7 @@ def __init__( "least-busy", "usage-based-routing", "latency-based-routing", + "cost-based-routing", ] = "simple-shuffle", ## DEBUGGING ## diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index f5777d6e7..2deca9258 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -39,6 +39,7 @@ const sidebars = { "proxy/demo", "proxy/configs", "proxy/reliability", + "proxy/cost_tracking", "proxy/users", "proxy/user_keys", "proxy/enterprise", @@ -50,8 +51,8 @@ const sidebars = { items: ["proxy/logging", "proxy/streaming_logging"], }, "proxy/team_based_routing", + "proxy/customer_routing", "proxy/ui", - "proxy/cost_tracking", "proxy/token_auth", { type: "category", @@ -131,9 +132,13 @@ const sidebars = { "providers/cohere", "providers/anyscale", "providers/huggingface", + "providers/watsonx", + "providers/predibase", + "providers/triton-inference-server", "providers/ollama", "providers/perplexity", "providers/groq", + "providers/deepseek", "providers/fireworks_ai", "providers/vllm", "providers/xinference", @@ -149,7 +154,7 @@ const sidebars = { "providers/openrouter", "providers/custom_openai_proxy", "providers/petals", - "providers/watsonx", + ], }, "proxy/custom_pricing", diff --git a/enterprise/enterprise_callbacks/generic_api_callback.py b/enterprise/enterprise_callbacks/generic_api_callback.py index 076c13d5e..cf1d22e8f 100644 --- a/enterprise/enterprise_callbacks/generic_api_callback.py +++ b/enterprise/enterprise_callbacks/generic_api_callback.py @@ -10,7 +10,6 @@ from litellm.caching import DualCache from typing import Literal, Union -dotenv.load_dotenv() # Loading env variables using dotenv import traceback @@ -19,8 +18,6 @@ import traceback import dotenv, os import requests - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/enterprise/utils.py b/enterprise/utils.py index 4a42dc996..90b14314c 100644 --- a/enterprise/utils.py +++ b/enterprise/utils.py @@ -1,6 +1,7 @@ # Enterprise Proxy Util Endpoints from litellm._logging import verbose_logger import collections +from datetime import datetime async def get_spend_by_tags(start_date=None, end_date=None, prisma_client=None): @@ -18,26 +19,33 @@ async def get_spend_by_tags(start_date=None, end_date=None, prisma_client=None): return response -async def ui_get_spend_by_tags(start_date=None, end_date=None, prisma_client=None): - response = await prisma_client.db.query_raw( - """ +async def ui_get_spend_by_tags(start_date: str, end_date: str, prisma_client): + + sql_query = """ SELECT jsonb_array_elements_text(request_tags) AS individual_request_tag, DATE(s."startTime") AS spend_date, COUNT(*) AS log_count, SUM(spend) AS total_spend FROM "LiteLLM_SpendLogs" s - WHERE s."startTime" >= current_date - interval '30 days' + WHERE + DATE(s."startTime") >= $1::date + AND DATE(s."startTime") <= $2::date GROUP BY individual_request_tag, spend_date - ORDER BY spend_date; - """ + ORDER BY spend_date + LIMIT 100; + """ + response = await prisma_client.db.query_raw( + sql_query, + start_date, + end_date, ) # print("tags - spend") # print(response) # Bar Chart 1 - Spend per tag - Top 10 tags by spend - total_spend_per_tag = collections.defaultdict(float) - total_requests_per_tag = collections.defaultdict(int) + total_spend_per_tag: collections.defaultdict = collections.defaultdict(float) + total_requests_per_tag: collections.defaultdict = collections.defaultdict(int) for row in response: tag_name = row["individual_request_tag"] tag_spend = row["total_spend"] @@ -49,15 +57,18 @@ async def ui_get_spend_by_tags(start_date=None, end_date=None, prisma_client=Non # convert to ui format ui_tags = [] for tag in sorted_tags: + current_spend = tag[1] + if current_spend is not None and isinstance(current_spend, float): + current_spend = round(current_spend, 4) ui_tags.append( { "name": tag[0], - "value": tag[1], + "spend": current_spend, "log_count": total_requests_per_tag[tag[0]], } ) - return {"top_10_tags": ui_tags} + return {"spend_per_tag": ui_tags} async def view_spend_logs_from_clickhouse( @@ -291,7 +302,7 @@ def _create_clickhouse_aggregate_tables(client=None, table_names=[]): def _forecast_daily_cost(data: list): - import requests + import requests # type: ignore from datetime import datetime, timedelta if len(data) == 0: diff --git a/index.yaml b/index.yaml new file mode 100644 index 000000000..8faeeeae6 --- /dev/null +++ b/index.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +entries: + litellm-helm: + - apiVersion: v2 + appVersion: v1.35.38 + created: "2024-05-06T10:22:24.384392-07:00" + dependencies: + - condition: db.deployStandalone + name: postgresql + repository: oci://registry-1.docker.io/bitnamicharts + version: '>=13.3.0' + - condition: redis.enabled + name: redis + repository: oci://registry-1.docker.io/bitnamicharts + version: '>=18.0.0' + description: Call all LLM APIs using the OpenAI format + digest: 60f0cfe9e7c1087437cb35f6fb7c43c3ab2be557b6d3aec8295381eb0dfa760f + name: litellm-helm + type: application + urls: + - litellm-helm-0.2.0.tgz + version: 0.2.0 + postgresql: + - annotations: + category: Database + images: | + - name: os-shell + image: docker.io/bitnami/os-shell:12-debian-12-r16 + - name: postgres-exporter + image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14 + - name: postgresql + image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6 + licenses: Apache-2.0 + apiVersion: v2 + appVersion: 16.2.0 + created: "2024-05-06T10:22:24.387717-07:00" + dependencies: + - name: common + repository: oci://registry-1.docker.io/bitnamicharts + tags: + - bitnami-common + version: 2.x.x + description: PostgreSQL (Postgres) is an open source object-relational database + known for reliability and data integrity. ACID-compliant, it supports foreign + keys, joins, views, triggers and stored procedures. + digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167 + home: https://bitnami.com + icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png + keywords: + - postgresql + - postgres + - database + - sql + - replication + - cluster + maintainers: + - name: VMware, Inc. + url: https://github.com/bitnami/charts + name: postgresql + sources: + - https://github.com/bitnami/charts/tree/main/bitnami/postgresql + urls: + - charts/postgresql-14.3.1.tgz + version: 14.3.1 + redis: + - annotations: + category: Database + images: | + - name: kubectl + image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3 + - name: os-shell + image: docker.io/bitnami/os-shell:12-debian-12-r16 + - name: redis + image: docker.io/bitnami/redis:7.2.4-debian-12-r9 + - name: redis-exporter + image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4 + - name: redis-sentinel + image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7 + licenses: Apache-2.0 + apiVersion: v2 + appVersion: 7.2.4 + created: "2024-05-06T10:22:24.391903-07:00" + dependencies: + - name: common + repository: oci://registry-1.docker.io/bitnamicharts + tags: + - bitnami-common + version: 2.x.x + description: Redis(R) is an open source, advanced key-value store. It is often + referred to as a data structure server since keys can contain strings, hashes, + lists, sets and sorted sets. + digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984 + home: https://bitnami.com + icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png + keywords: + - redis + - keyvalue + - database + maintainers: + - name: VMware, Inc. + url: https://github.com/bitnami/charts + name: redis + sources: + - https://github.com/bitnami/charts/tree/main/bitnami/redis + urls: + - charts/redis-18.19.1.tgz + version: 18.19.1 +generated: "2024-05-06T10:22:24.375026-07:00" diff --git a/litellm-helm-0.2.0.tgz b/litellm-helm-0.2.0.tgz new file mode 100644 index 000000000..9d0947348 Binary files /dev/null and b/litellm-helm-0.2.0.tgz differ diff --git a/litellm/__init__.py b/litellm/__init__.py index dc640f0e9..16395b27f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1,3 +1,7 @@ +### Hide pydantic namespace conflict warnings globally ### +import warnings + +warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*") ### INIT VARIABLES ### import threading, requests, os from typing import Callable, List, Optional, Dict, Union, Any, Literal @@ -67,13 +71,16 @@ azure_key: Optional[str] = None anthropic_key: Optional[str] = None replicate_key: Optional[str] = None cohere_key: Optional[str] = None +clarifai_key: Optional[str] = None maritalk_key: Optional[str] = None ai21_key: Optional[str] = None ollama_key: Optional[str] = None openrouter_key: Optional[str] = None +predibase_key: Optional[str] = None huggingface_key: Optional[str] = None vertex_project: Optional[str] = None vertex_location: Optional[str] = None +predibase_tenant_id: Optional[str] = None togetherai_api_key: Optional[str] = None cloudflare_api_key: Optional[str] = None baseten_key: Optional[str] = None @@ -95,6 +102,9 @@ blocked_user_list: Optional[Union[str, List]] = None banned_keywords_list: Optional[Union[str, List]] = None llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" ################## +### PREVIEW FEATURES ### +enable_preview_features: bool = False +################## logging: bool = True caching: bool = ( False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 @@ -361,6 +371,7 @@ openai_compatible_endpoints: List = [ "api.deepinfra.com/v1/openai", "api.mistral.ai/v1", "api.groq.com/openai/v1", + "api.deepseek.com/v1", "api.together.xyz/v1", ] @@ -369,6 +380,7 @@ openai_compatible_providers: List = [ "anyscale", "mistral", "groq", + "deepseek", "deepinfra", "perplexity", "xinference", @@ -393,6 +405,73 @@ replicate_models: List = [ "replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad", ] +clarifai_models: List = [ + "clarifai/meta.Llama-3.Llama-3-8B-Instruct", + "clarifai/gcp.generate.gemma-1_1-7b-it", + "clarifai/mistralai.completion.mixtral-8x22B", + "clarifai/cohere.generate.command-r-plus", + "clarifai/databricks.drbx.dbrx-instruct", + "clarifai/mistralai.completion.mistral-large", + "clarifai/mistralai.completion.mistral-medium", + "clarifai/mistralai.completion.mistral-small", + "clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1", + "clarifai/gcp.generate.gemma-2b-it", + "clarifai/gcp.generate.gemma-7b-it", + "clarifai/deci.decilm.deciLM-7B-instruct", + "clarifai/mistralai.completion.mistral-7B-Instruct", + "clarifai/gcp.generate.gemini-pro", + "clarifai/anthropic.completion.claude-v1", + "clarifai/anthropic.completion.claude-instant-1_2", + "clarifai/anthropic.completion.claude-instant", + "clarifai/anthropic.completion.claude-v2", + "clarifai/anthropic.completion.claude-2_1", + "clarifai/meta.Llama-2.codeLlama-70b-Python", + "clarifai/meta.Llama-2.codeLlama-70b-Instruct", + "clarifai/openai.completion.gpt-3_5-turbo-instruct", + "clarifai/meta.Llama-2.llama2-7b-chat", + "clarifai/meta.Llama-2.llama2-13b-chat", + "clarifai/meta.Llama-2.llama2-70b-chat", + "clarifai/openai.chat-completion.gpt-4-turbo", + "clarifai/microsoft.text-generation.phi-2", + "clarifai/meta.Llama-2.llama2-7b-chat-vllm", + "clarifai/upstage.solar.solar-10_7b-instruct", + "clarifai/openchat.openchat.openchat-3_5-1210", + "clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B", + "clarifai/gcp.generate.text-bison", + "clarifai/meta.Llama-2.llamaGuard-7b", + "clarifai/fblgit.una-cybertron.una-cybertron-7b-v2", + "clarifai/openai.chat-completion.GPT-4", + "clarifai/openai.chat-completion.GPT-3_5-turbo", + "clarifai/ai21.complete.Jurassic2-Grande", + "clarifai/ai21.complete.Jurassic2-Grande-Instruct", + "clarifai/ai21.complete.Jurassic2-Jumbo-Instruct", + "clarifai/ai21.complete.Jurassic2-Jumbo", + "clarifai/ai21.complete.Jurassic2-Large", + "clarifai/cohere.generate.cohere-generate-command", + "clarifai/wizardlm.generate.wizardCoder-Python-34B", + "clarifai/wizardlm.generate.wizardLM-70B", + "clarifai/tiiuae.falcon.falcon-40b-instruct", + "clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat", + "clarifai/gcp.generate.code-gecko", + "clarifai/gcp.generate.code-bison", + "clarifai/mistralai.completion.mistral-7B-OpenOrca", + "clarifai/mistralai.completion.openHermes-2-mistral-7B", + "clarifai/wizardlm.generate.wizardLM-13B", + "clarifai/huggingface-research.zephyr.zephyr-7B-alpha", + "clarifai/wizardlm.generate.wizardCoder-15B", + "clarifai/microsoft.text-generation.phi-1_5", + "clarifai/databricks.Dolly-v2.dolly-v2-12b", + "clarifai/bigcode.code.StarCoder", + "clarifai/salesforce.xgen.xgen-7b-8k-instruct", + "clarifai/mosaicml.mpt.mpt-7b-instruct", + "clarifai/anthropic.completion.claude-3-opus", + "clarifai/anthropic.completion.claude-3-sonnet", + "clarifai/gcp.generate.gemini-1_5-pro", + "clarifai/gcp.generate.imagen-2", + "clarifai/salesforce.blip.general-english-image-caption-blip-2", +] + + huggingface_models: List = [ "meta-llama/Llama-2-7b-hf", "meta-llama/Llama-2-7b-chat-hf", @@ -498,6 +577,7 @@ provider_list: List = [ "text-completion-openai", "cohere", "cohere_chat", + "clarifai", "anthropic", "replicate", "huggingface", @@ -523,12 +603,15 @@ provider_list: List = [ "anyscale", "mistral", "groq", + "deepseek", "maritalk", "voyage", "cloudflare", "xinference", "fireworks_ai", "watsonx", + "triton", + "predibase", "custom", # custom apis ] @@ -605,7 +688,6 @@ all_embedding_models = ( ####### IMAGE GENERATION MODELS ################### openai_image_generation_models = ["dall-e-2", "dall-e-3"] - from .timeout import timeout from .utils import ( client, @@ -638,12 +720,15 @@ from .utils import ( get_secret, get_supported_openai_params, get_api_base, + get_first_chars_messages, ) from .llms.huggingface_restapi import HuggingfaceConfig from .llms.anthropic import AnthropicConfig +from .llms.predibase import PredibaseConfig from .llms.anthropic_text import AnthropicTextConfig from .llms.replicate import ReplicateConfig from .llms.cohere import CohereConfig +from .llms.clarifai import ClarifaiConfig from .llms.ai21 import AI21Config from .llms.together_ai import TogetherAIConfig from .llms.cloudflare import CloudflareConfig @@ -658,6 +743,7 @@ from .llms.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.ollama_chat import OllamaChatConfig from .llms.maritalk import MaritTalkConfig +from .llms.bedrock_httpx import AmazonCohereChatConfig from .llms.bedrock import ( AmazonTitanConfig, AmazonAI21Config, @@ -669,7 +755,7 @@ from .llms.bedrock import ( AmazonMistralConfig, AmazonBedrockGlobalConfig, ) -from .llms.openai import OpenAIConfig, OpenAITextCompletionConfig +from .llms.openai import OpenAIConfig, OpenAITextCompletionConfig, MistralConfig from .llms.azure import AzureOpenAIConfig, AzureOpenAIError from .llms.watsonx import IBMWatsonXAIConfig from .main import * # type: ignore @@ -694,3 +780,4 @@ from .exceptions import ( from .budget_manager import BudgetManager from .proxy.proxy_cli import run_server from .router import Router +from .assistants.main import * diff --git a/litellm/_redis.py b/litellm/_redis.py index d7789472c..d72016dcd 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -10,8 +10,8 @@ # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation import os import inspect -import redis, litellm -import redis.asyncio as async_redis +import redis, litellm # type: ignore +import redis.asyncio as async_redis # type: ignore from typing import List, Optional diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py new file mode 100644 index 000000000..25d2433d7 --- /dev/null +++ b/litellm/assistants/main.py @@ -0,0 +1,495 @@ +# What is this? +## Main file for assistants API logic +from typing import Iterable +import os +import litellm +from openai import OpenAI +from litellm import client +from litellm.utils import supports_httpx_timeout +from ..llms.openai import OpenAIAssistantsAPI +from ..types.llms.openai import * +from ..types.router import * + +####### ENVIRONMENT VARIABLES ################### +openai_assistants_api = OpenAIAssistantsAPI() + +### ASSISTANTS ### + + +def get_assistants( + custom_llm_provider: Literal["openai"], + client: Optional[OpenAI] = None, + **kwargs, +) -> SyncCursorPage[Assistant]: + optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + response: Optional[SyncCursorPage[Assistant]] = None + if custom_llm_provider == "openai": + api_base = ( + optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + response = openai_assistants_api.get_assistants( + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + client=client, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'get_assistants'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + + +### THREADS ### + + +def create_thread( + custom_llm_provider: Literal["openai"], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]] = None, + metadata: Optional[dict] = None, + tool_resources: Optional[OpenAICreateThreadParamsToolResources] = None, + client: Optional[OpenAI] = None, + **kwargs, +) -> Thread: + """ + - get the llm provider + - if openai - route it there + - pass through relevant params + + ``` + from litellm import create_thread + + create_thread( + custom_llm_provider="openai", + ### OPTIONAL ### + messages = { + "role": "user", + "content": "Hello, what is AI?" + }, + { + "role": "user", + "content": "How does AI work? Explain it in simple terms." + }] + ) + ``` + """ + optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + response: Optional[Thread] = None + if custom_llm_provider == "openai": + api_base = ( + optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + response = openai_assistants_api.create_thread( + messages=messages, + metadata=metadata, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + client=client, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_thread'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + + +def get_thread( + custom_llm_provider: Literal["openai"], + thread_id: str, + client: Optional[OpenAI] = None, + **kwargs, +) -> Thread: + """Get the thread object, given a thread_id""" + optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + response: Optional[Thread] = None + if custom_llm_provider == "openai": + api_base = ( + optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + response = openai_assistants_api.get_thread( + thread_id=thread_id, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + client=client, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'get_thread'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + + +### MESSAGES ### + + +def add_message( + custom_llm_provider: Literal["openai"], + thread_id: str, + role: Literal["user", "assistant"], + content: str, + attachments: Optional[List[Attachment]] = None, + metadata: Optional[dict] = None, + client: Optional[OpenAI] = None, + **kwargs, +) -> OpenAIMessage: + ### COMMON OBJECTS ### + message_data = MessageData( + role=role, content=content, attachments=attachments, metadata=metadata + ) + optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + response: Optional[OpenAIMessage] = None + if custom_llm_provider == "openai": + api_base = ( + optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + response = openai_assistants_api.add_message( + thread_id=thread_id, + message_data=message_data, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + client=client, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_thread'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + + return response + + +def get_messages( + custom_llm_provider: Literal["openai"], + thread_id: str, + client: Optional[OpenAI] = None, + **kwargs, +) -> SyncCursorPage[OpenAIMessage]: + optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + response: Optional[SyncCursorPage[OpenAIMessage]] = None + if custom_llm_provider == "openai": + api_base = ( + optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + response = openai_assistants_api.get_messages( + thread_id=thread_id, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + client=client, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'get_messages'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + + return response + + +### RUNS ### + + +def run_thread( + custom_llm_provider: Literal["openai"], + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str] = None, + instructions: Optional[str] = None, + metadata: Optional[dict] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + tools: Optional[Iterable[AssistantToolParam]] = None, + client: Optional[OpenAI] = None, + **kwargs, +) -> Run: + """Run a given thread + assistant.""" + optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + response: Optional[Run] = None + if custom_llm_provider == "openai": + api_base = ( + optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + response = openai_assistants_api.run_thread( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + stream=stream, + tools=tools, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + client=client, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'run_thread'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response diff --git a/litellm/budget_manager.py b/litellm/budget_manager.py index 841015753..9ef4bfafa 100644 --- a/litellm/budget_manager.py +++ b/litellm/budget_manager.py @@ -10,7 +10,7 @@ import os, json, time import litellm from litellm.utils import ModelResponse -import requests, threading +import requests, threading # type: ignore from typing import Optional, Union, Literal diff --git a/litellm/caching.py b/litellm/caching.py index 86e3ef40d..ccb62b882 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -106,7 +106,7 @@ class InMemoryCache(BaseCache): return_val.append(val) return return_val - async def async_increment(self, key, value: int, **kwargs) -> int: + async def async_increment(self, key, value: float, **kwargs) -> float: # get the value init_value = await self.async_get_cache(key=key) or 0 value = init_value + value @@ -177,11 +177,18 @@ class RedisCache(BaseCache): try: # asyncio.get_running_loop().create_task(self.ping()) result = asyncio.get_running_loop().create_task(self.ping()) - except Exception: - pass + except Exception as e: + verbose_logger.error( + "Error connecting to Async Redis client", extra={"error": str(e)} + ) ### SYNC HEALTH PING ### - self.redis_client.ping() + try: + self.redis_client.ping() + except Exception as e: + verbose_logger.error( + "Error connecting to Sync Redis client", extra={"error": str(e)} + ) def init_async_client(self): from ._redis import get_redis_async_client @@ -366,11 +373,12 @@ class RedisCache(BaseCache): print_verbose( f"Set ASYNC Redis Cache PIPELINE: key: {cache_key}\nValue {cache_value}\nttl={ttl}" ) + json_cache_value = json.dumps(cache_value) # Set the value with a TTL if it's provided. if ttl is not None: - pipe.setex(cache_key, ttl, json.dumps(cache_value)) + pipe.setex(cache_key, ttl, json_cache_value) else: - pipe.set(cache_key, json.dumps(cache_value)) + pipe.set(cache_key, json_cache_value) # Execute the pipeline and return the results. results = await pipe.execute() @@ -416,12 +424,12 @@ class RedisCache(BaseCache): if len(self.redis_batch_writing_buffer) >= self.redis_flush_size: await self.flush_cache_buffer() # logging done in here - async def async_increment(self, key, value: int, **kwargs) -> int: + async def async_increment(self, key, value: float, **kwargs) -> float: _redis_client = self.init_async_client() start_time = time.time() try: async with _redis_client as redis_client: - result = await redis_client.incr(name=key, amount=value) + result = await redis_client.incrbyfloat(name=key, amount=value) ## LOGGING ## end_time = time.time() _duration = end_time - start_time @@ -803,9 +811,7 @@ class RedisSemanticCache(BaseCache): # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) # create an embedding for prompt embedding_response = litellm.embedding( @@ -840,9 +846,7 @@ class RedisSemanticCache(BaseCache): # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) # convert to embedding embedding_response = litellm.embedding( @@ -902,9 +906,7 @@ class RedisSemanticCache(BaseCache): # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) # create an embedding for prompt router_model_names = ( [m["model_name"] for m in llm_model_list] @@ -957,9 +959,7 @@ class RedisSemanticCache(BaseCache): # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) router_model_names = ( [m["model_name"] for m in llm_model_list] @@ -1375,18 +1375,41 @@ class DualCache(BaseCache): print_verbose(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") traceback.print_exc() + async def async_batch_set_cache( + self, cache_list: list, local_only: bool = False, **kwargs + ): + """ + Batch write values to the cache + """ + print_verbose( + f"async batch set cache: cache keys: {cache_list}; local_only: {local_only}" + ) + try: + if self.in_memory_cache is not None: + await self.in_memory_cache.async_set_cache_pipeline( + cache_list=cache_list, **kwargs + ) + + if self.redis_cache is not None and local_only == False: + await self.redis_cache.async_set_cache_pipeline( + cache_list=cache_list, ttl=kwargs.get("ttl", None) + ) + except Exception as e: + print_verbose(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") + traceback.print_exc() + async def async_increment_cache( - self, key, value: int, local_only: bool = False, **kwargs - ) -> int: + self, key, value: float, local_only: bool = False, **kwargs + ) -> float: """ Key - the key in cache - Value - int - the value you want to increment by + Value - float - the value you want to increment by - Returns - int - the incremented value + Returns - float - the incremented value """ try: - result: int = value + result: float = value if self.in_memory_cache is not None: result = await self.in_memory_cache.async_increment( key, value, **kwargs diff --git a/litellm/exceptions.py b/litellm/exceptions.py index d8b0a7c55..d239f1e12 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,25 +9,12 @@ ## LiteLLM versions of the OpenAI Exception Types -from openai import ( - AuthenticationError, - BadRequestError, - NotFoundError, - RateLimitError, - APIStatusError, - OpenAIError, - APIError, - APITimeoutError, - APIConnectionError, - APIResponseValidationError, - UnprocessableEntityError, - PermissionDeniedError, -) +import openai import httpx from typing import Optional -class AuthenticationError(AuthenticationError): # type: ignore +class AuthenticationError(openai.AuthenticationError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 401 self.message = message @@ -39,7 +26,7 @@ class AuthenticationError(AuthenticationError): # type: ignore # raise when invalid models passed, example gpt-8 -class NotFoundError(NotFoundError): # type: ignore +class NotFoundError(openai.NotFoundError): # type: ignore def __init__(self, message, model, llm_provider, response: httpx.Response): self.status_code = 404 self.message = message @@ -50,7 +37,7 @@ class NotFoundError(NotFoundError): # type: ignore ) # Call the base class constructor with the parameters it needs -class BadRequestError(BadRequestError): # type: ignore +class BadRequestError(openai.BadRequestError): # type: ignore def __init__( self, message, model, llm_provider, response: Optional[httpx.Response] = None ): @@ -69,7 +56,7 @@ class BadRequestError(BadRequestError): # type: ignore ) # Call the base class constructor with the parameters it needs -class UnprocessableEntityError(UnprocessableEntityError): # type: ignore +class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore def __init__(self, message, model, llm_provider, response: httpx.Response): self.status_code = 422 self.message = message @@ -80,7 +67,7 @@ class UnprocessableEntityError(UnprocessableEntityError): # type: ignore ) # Call the base class constructor with the parameters it needs -class Timeout(APITimeoutError): # type: ignore +class Timeout(openai.APITimeoutError): # type: ignore def __init__(self, message, model, llm_provider): request = httpx.Request(method="POST", url="https://api.openai.com/v1") super().__init__( @@ -96,7 +83,7 @@ class Timeout(APITimeoutError): # type: ignore return str(self.message) -class PermissionDeniedError(PermissionDeniedError): # type:ignore +class PermissionDeniedError(openai.PermissionDeniedError): # type:ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 403 self.message = message @@ -107,7 +94,7 @@ class PermissionDeniedError(PermissionDeniedError): # type:ignore ) # Call the base class constructor with the parameters it needs -class RateLimitError(RateLimitError): # type: ignore +class RateLimitError(openai.RateLimitError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 429 self.message = message @@ -148,7 +135,7 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore ) # Call the base class constructor with the parameters it needs -class ServiceUnavailableError(APIStatusError): # type: ignore +class ServiceUnavailableError(openai.APIStatusError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 503 self.message = message @@ -160,7 +147,7 @@ class ServiceUnavailableError(APIStatusError): # type: ignore # raise this when the API returns an invalid response object - https://github.com/openai/openai-python/blob/1be14ee34a0f8e42d3f9aa5451aa4cb161f1781f/openai/api_requestor.py#L401 -class APIError(APIError): # type: ignore +class APIError(openai.APIError): # type: ignore def __init__( self, status_code, message, llm_provider, model, request: httpx.Request ): @@ -172,7 +159,7 @@ class APIError(APIError): # type: ignore # raised if an invalid request (not get, delete, put, post) is made -class APIConnectionError(APIConnectionError): # type: ignore +class APIConnectionError(openai.APIConnectionError): # type: ignore def __init__(self, message, llm_provider, model, request: httpx.Request): self.message = message self.llm_provider = llm_provider @@ -182,7 +169,7 @@ class APIConnectionError(APIConnectionError): # type: ignore # raised if an invalid request (not get, delete, put, post) is made -class APIResponseValidationError(APIResponseValidationError): # type: ignore +class APIResponseValidationError(openai.APIResponseValidationError): # type: ignore def __init__(self, message, llm_provider, model): self.message = message self.llm_provider = llm_provider @@ -192,7 +179,7 @@ class APIResponseValidationError(APIResponseValidationError): # type: ignore super().__init__(response=response, body=None, message=message) -class OpenAIError(OpenAIError): # type: ignore +class OpenAIError(openai.OpenAIError): # type: ignore def __init__(self, original_exception): self.status_code = original_exception.http_status super().__init__( @@ -214,7 +201,7 @@ class BudgetExceededError(Exception): ## DEPRECATED ## -class InvalidRequestError(BadRequestError): # type: ignore +class InvalidRequestError(openai.BadRequestError): # type: ignore def __init__(self, message, model, llm_provider): self.status_code = 400 self.message = message diff --git a/litellm/integrations/aispend.py b/litellm/integrations/aispend.py index 2015d45dd..2fe8ea0df 100644 --- a/litellm/integrations/aispend.py +++ b/litellm/integrations/aispend.py @@ -1,9 +1,6 @@ #### What this does #### # On success + failure, log events to aispend.io import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime diff --git a/litellm/integrations/athina.py b/litellm/integrations/athina.py index 897cf6c8d..660dd51ef 100644 --- a/litellm/integrations/athina.py +++ b/litellm/integrations/athina.py @@ -4,18 +4,30 @@ import datetime class AthinaLogger: def __init__(self): import os + self.athina_api_key = os.getenv("ATHINA_API_KEY") self.headers = { "athina-api-key": self.athina_api_key, - "Content-Type": "application/json" + "Content-Type": "application/json", } self.athina_logging_url = "https://log.athina.ai/api/v1/log/inference" - self.additional_keys = ["environment", "prompt_slug", "customer_id", "customer_user_id", "session_id", "external_reference_id", "context", "expected_response", "user_query"] + self.additional_keys = [ + "environment", + "prompt_slug", + "customer_id", + "customer_user_id", + "session_id", + "external_reference_id", + "context", + "expected_response", + "user_query", + ] def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): - import requests + import requests # type: ignore import json import traceback + try: response_json = response_obj.model_dump() if response_obj else {} data = { @@ -23,32 +35,51 @@ class AthinaLogger: "request": kwargs, "response": response_json, "prompt_tokens": response_json.get("usage", {}).get("prompt_tokens"), - "completion_tokens": response_json.get("usage", {}).get("completion_tokens"), + "completion_tokens": response_json.get("usage", {}).get( + "completion_tokens" + ), "total_tokens": response_json.get("usage", {}).get("total_tokens"), } - - if type(end_time) == datetime.datetime and type(start_time) == datetime.datetime: - data["response_time"] = int((end_time - start_time).total_seconds() * 1000) + + if ( + type(end_time) == datetime.datetime + and type(start_time) == datetime.datetime + ): + data["response_time"] = int( + (end_time - start_time).total_seconds() * 1000 + ) if "messages" in kwargs: data["prompt"] = kwargs.get("messages", None) # Directly add tools or functions if present optional_params = kwargs.get("optional_params", {}) - data.update((k, v) for k, v in optional_params.items() if k in ["tools", "functions"]) + data.update( + (k, v) + for k, v in optional_params.items() + if k in ["tools", "functions"] + ) # Add additional metadata keys - metadata = kwargs.get("litellm_params", {}).get("metadata", {}) + metadata = kwargs.get("litellm_params", {}).get("metadata", {}) if metadata: for key in self.additional_keys: if key in metadata: data[key] = metadata[key] - response = requests.post(self.athina_logging_url, headers=self.headers, data=json.dumps(data, default=str)) + response = requests.post( + self.athina_logging_url, + headers=self.headers, + data=json.dumps(data, default=str), + ) if response.status_code != 200: - print_verbose(f"Athina Logger Error - {response.text}, {response.status_code}") + print_verbose( + f"Athina Logger Error - {response.text}, {response.status_code}" + ) else: print_verbose(f"Athina Logger Succeeded - {response.text}") except Exception as e: - print_verbose(f"Athina Logger Error - {e}, Stack trace: {traceback.format_exc()}") - pass \ No newline at end of file + print_verbose( + f"Athina Logger Error - {e}, Stack trace: {traceback.format_exc()}" + ) + pass diff --git a/litellm/integrations/berrispend.py b/litellm/integrations/berrispend.py index 7d91ffca7..7d30b706c 100644 --- a/litellm/integrations/berrispend.py +++ b/litellm/integrations/berrispend.py @@ -1,9 +1,8 @@ #### What this does #### # On success + failure, log events to aispend.io import dotenv, os -import requests +import requests # type: ignore -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index d5000e5c4..0c38b8626 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -3,14 +3,11 @@ #### What this does #### # On success, logs events to Promptlayer import dotenv, os -import requests from litellm.proxy._types import UserAPIKeyAuth from litellm.caching import DualCache from typing import Literal, Union - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback @@ -19,8 +16,6 @@ import traceback import dotenv, os import requests - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index b288036ad..d50882592 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -1,14 +1,11 @@ #### What this does #### # On success, logs events to Promptlayer import dotenv, os -import requests from litellm.proxy._types import UserAPIKeyAuth from litellm.caching import DualCache from typing import Literal, Union, Optional - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/integrations/datadog.py b/litellm/integrations/datadog.py index f5db5bf1f..6d5e08faf 100644 --- a/litellm/integrations/datadog.py +++ b/litellm/integrations/datadog.py @@ -2,9 +2,7 @@ # On success + failure, log events to Supabase import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv +import requests # type: ignore import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 2ed6c3f9f..21ccabe4b 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -2,9 +2,7 @@ # On success + failure, log events to Supabase import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv +import requests # type: ignore import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/greenscale.py b/litellm/integrations/greenscale.py index 3ff808ddb..78190d69d 100644 --- a/litellm/integrations/greenscale.py +++ b/litellm/integrations/greenscale.py @@ -1,15 +1,17 @@ -import requests +import requests # type: ignore import json import traceback from datetime import datetime, timezone + class GreenscaleLogger: def __init__(self): import os + self.greenscale_api_key = os.getenv("GREENSCALE_API_KEY") self.headers = { "api-key": self.greenscale_api_key, - "Content-Type": "application/json" + "Content-Type": "application/json", } self.greenscale_logging_url = os.getenv("GREENSCALE_ENDPOINT") @@ -19,33 +21,48 @@ class GreenscaleLogger: data = { "modelId": kwargs.get("model"), "inputTokenCount": response_json.get("usage", {}).get("prompt_tokens"), - "outputTokenCount": response_json.get("usage", {}).get("completion_tokens"), + "outputTokenCount": response_json.get("usage", {}).get( + "completion_tokens" + ), } - data["timestamp"] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') - - if type(end_time) == datetime and type(start_time) == datetime: - data["invocationLatency"] = int((end_time - start_time).total_seconds() * 1000) + data["timestamp"] = datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + if type(end_time) == datetime and type(start_time) == datetime: + data["invocationLatency"] = int( + (end_time - start_time).total_seconds() * 1000 + ) # Add additional metadata keys to tags tags = [] metadata = kwargs.get("litellm_params", {}).get("metadata", {}) for key, value in metadata.items(): - if key.startswith("greenscale"): + if key.startswith("greenscale"): if key == "greenscale_project": data["project"] = value elif key == "greenscale_application": data["application"] = value else: - tags.append({"key": key.replace("greenscale_", ""), "value": str(value)}) - + tags.append( + {"key": key.replace("greenscale_", ""), "value": str(value)} + ) + data["tags"] = tags - response = requests.post(self.greenscale_logging_url, headers=self.headers, data=json.dumps(data, default=str)) + response = requests.post( + self.greenscale_logging_url, + headers=self.headers, + data=json.dumps(data, default=str), + ) if response.status_code != 200: - print_verbose(f"Greenscale Logger Error - {response.text}, {response.status_code}") + print_verbose( + f"Greenscale Logger Error - {response.text}, {response.status_code}" + ) else: print_verbose(f"Greenscale Logger Succeeded - {response.text}") except Exception as e: - print_verbose(f"Greenscale Logger Error - {e}, Stack trace: {traceback.format_exc()}") - pass \ No newline at end of file + print_verbose( + f"Greenscale Logger Error - {e}, Stack trace: {traceback.format_exc()}" + ) + pass diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index cb8663773..85e73258e 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -1,10 +1,8 @@ #### What this does #### # On success, logs events to Helicone import dotenv, os -import requests +import requests # type: ignore import litellm - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 304edbbc9..f4a581eb9 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -1,8 +1,6 @@ #### What this does #### # On success, logs events to Langfuse -import dotenv, os - -dotenv.load_dotenv() # Loading env variables using dotenv +import os import copy import traceback from packaging.version import Version @@ -262,6 +260,23 @@ class LangFuseLogger: try: tags = [] + try: + metadata = copy.deepcopy( + metadata + ) # Avoid modifying the original metadata + except: + new_metadata = {} + for key, value in metadata.items(): + if ( + isinstance(value, list) + or isinstance(value, dict) + or isinstance(value, str) + or isinstance(value, int) + or isinstance(value, float) + ): + new_metadata[key] = copy.deepcopy(value) + metadata = new_metadata + supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3") supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3") supports_costs = Version(langfuse.version.__version__) >= Version("2.7.3") @@ -272,36 +287,9 @@ class LangFuseLogger: print_verbose(f"Langfuse Layer Logging - logging to langfuse v2 ") if supports_tags: - metadata_tags = metadata.get("tags", []) + metadata_tags = metadata.pop("tags", []) tags = metadata_tags - trace_name = metadata.get("trace_name", None) - trace_id = metadata.get("trace_id", None) - existing_trace_id = metadata.get("existing_trace_id", None) - if trace_name is None and existing_trace_id is None: - # just log `litellm-{call_type}` as the trace name - ## DO NOT SET TRACE_NAME if trace-id set. this can lead to overwriting of past traces. - trace_name = f"litellm-{kwargs.get('call_type', 'completion')}" - - if existing_trace_id is not None: - trace_params = {"id": existing_trace_id} - else: # don't overwrite an existing trace - trace_params = { - "name": trace_name, - "input": input, - "user_id": metadata.get("trace_user_id", user_id), - "id": trace_id, - "session_id": metadata.get("session_id", None), - } - - if level == "ERROR": - trace_params["status_message"] = output - else: - trace_params["output"] = output - - cost = kwargs.get("response_cost", None) - print_verbose(f"trace: {cost}") - # Clean Metadata before logging - never log raw metadata # the raw metadata can contain circular references which leads to infinite recursion # we clean out all extra litellm metadata params before logging @@ -328,6 +316,77 @@ class LangFuseLogger: else: clean_metadata[key] = value + session_id = clean_metadata.pop("session_id", None) + trace_name = clean_metadata.pop("trace_name", None) + trace_id = clean_metadata.pop("trace_id", None) + existing_trace_id = clean_metadata.pop("existing_trace_id", None) + update_trace_keys = clean_metadata.pop("update_trace_keys", []) + debug = clean_metadata.pop("debug_langfuse", None) + mask_input = clean_metadata.pop("mask_input", False) + mask_output = clean_metadata.pop("mask_output", False) + + if trace_name is None and existing_trace_id is None: + # just log `litellm-{call_type}` as the trace name + ## DO NOT SET TRACE_NAME if trace-id set. this can lead to overwriting of past traces. + trace_name = f"litellm-{kwargs.get('call_type', 'completion')}" + + if existing_trace_id is not None: + trace_params = {"id": existing_trace_id} + + # Update the following keys for this trace + for metadata_param_key in update_trace_keys: + trace_param_key = metadata_param_key.replace("trace_", "") + if trace_param_key not in trace_params: + updated_trace_value = clean_metadata.pop( + metadata_param_key, None + ) + if updated_trace_value is not None: + trace_params[trace_param_key] = updated_trace_value + + # Pop the trace specific keys that would have been popped if there were a new trace + for key in list( + filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) + ): + clean_metadata.pop(key, None) + + # Special keys that are found in the function arguments and not the metadata + if "input" in update_trace_keys: + trace_params["input"] = input if not mask_input else "redacted-by-litellm" + if "output" in update_trace_keys: + trace_params["output"] = output if not mask_output else "redacted-by-litellm" + else: # don't overwrite an existing trace + trace_params = { + "id": trace_id, + "name": trace_name, + "session_id": session_id, + "input": input if not mask_input else "redacted-by-litellm", + "version": clean_metadata.pop( + "trace_version", clean_metadata.get("version", None) + ), # If provided just version, it will applied to the trace as well, if applied a trace version it will take precedence + "user_id": user_id, + } + for key in list( + filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) + ): + trace_params[key.replace("trace_", "")] = clean_metadata.pop( + key, None + ) + + if level == "ERROR": + trace_params["status_message"] = output + else: + trace_params["output"] = output if not mask_output else "redacted-by-litellm" + + if debug == True or (isinstance(debug, str) and debug.lower() == "true"): + if "metadata" in trace_params: + # log the raw_metadata in the trace + trace_params["metadata"]["metadata_passed_to_litellm"] = metadata + else: + trace_params["metadata"] = {"metadata_passed_to_litellm": metadata} + + cost = kwargs.get("response_cost", None) + print_verbose(f"trace: {cost}") + if ( litellm._langfuse_default_tags is not None and isinstance(litellm._langfuse_default_tags, list) @@ -375,7 +434,6 @@ class LangFuseLogger: "url": url, "headers": clean_headers, } - trace = self.Langfuse.trace(**trace_params) generation_id = None @@ -387,7 +445,7 @@ class LangFuseLogger: "completion_tokens": response_obj["usage"]["completion_tokens"], "total_cost": cost if supports_costs else None, } - generation_name = metadata.get("generation_name", None) + generation_name = clean_metadata.pop("generation_name", None) if generation_name is None: # just log `litellm-{call_type}` as the generation name generation_name = f"litellm-{kwargs.get('call_type', 'completion')}" @@ -402,20 +460,43 @@ class LangFuseLogger: generation_params = { "name": generation_name, - "id": metadata.get("generation_id", generation_id), + "id": clean_metadata.pop("generation_id", generation_id), "start_time": start_time, "end_time": end_time, "model": kwargs["model"], "model_parameters": optional_params, - "input": input, - "output": output, + "input": input if not mask_input else "redacted-by-litellm", + "output": output if not mask_output else "redacted-by-litellm", "usage": usage, "metadata": clean_metadata, "level": level, + "version": clean_metadata.pop("version", None), } if supports_prompt: - generation_params["prompt"] = metadata.get("prompt", None) + user_prompt = clean_metadata.pop("prompt", None) + if user_prompt is None: + pass + elif isinstance(user_prompt, dict): + from langfuse.model import ( + TextPromptClient, + ChatPromptClient, + Prompt_Text, + Prompt_Chat, + ) + + if user_prompt.get("type", "") == "chat": + _prompt_chat = Prompt_Chat(**user_prompt) + generation_params["prompt"] = ChatPromptClient( + prompt=_prompt_chat + ) + elif user_prompt.get("type", "") == "text": + _prompt_text = Prompt_Text(**user_prompt) + generation_params["prompt"] = TextPromptClient( + prompt=_prompt_text + ) + else: + generation_params["prompt"] = user_prompt if output is not None and isinstance(output, str) and level == "ERROR": generation_params["status_message"] = output @@ -426,7 +507,7 @@ class LangFuseLogger: ) generation_client = trace.generation(**generation_params) - + return generation_client.trace_id, generation_id except Exception as e: verbose_logger.debug(f"Langfuse Layer Error - {traceback.format_exc()}") diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 415f3d2d2..92e440215 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -1,15 +1,12 @@ #### What this does #### # On success, logs events to Langsmith -import dotenv, os -import requests -import requests +import dotenv, os # type: ignore +import requests # type: ignore from datetime import datetime - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import asyncio import types -from pydantic import BaseModel +from pydantic import BaseModel # type: ignore def is_serializable(value): @@ -79,8 +76,6 @@ class LangsmithLogger: except: response_obj = response_obj.dict() # type: ignore - print(f"response_obj: {response_obj}") - data = { "name": run_name, "run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain" @@ -90,7 +85,6 @@ class LangsmithLogger: "start_time": start_time, "end_time": end_time, } - print(f"data: {data}") response = requests.post( "https://api.smith.langchain.com/runs", diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 6ddf2ca59..2e16e44a1 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -2,14 +2,10 @@ # On success + failure, log events to lunary.ai from datetime import datetime, timezone import traceback -import dotenv import importlib -import sys import packaging -dotenv.load_dotenv() - # convert to {completion: xx, tokens: xx} def parse_usage(usage): @@ -18,13 +14,33 @@ def parse_usage(usage): "prompt": usage["prompt_tokens"] if "prompt_tokens" in usage else 0, } +def parse_tool_calls(tool_calls): + if tool_calls is None: + return None + + def clean_tool_call(tool_call): + + serialized = { + "type": tool_call.type, + "id": tool_call.id, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + } + } + + return serialized + + return [clean_tool_call(tool_call) for tool_call in tool_calls] + def parse_messages(input): + if input is None: return None def clean_message(message): - # if is strin, return as is + # if is string, return as is if isinstance(message, str): return message @@ -38,9 +54,7 @@ def parse_messages(input): # Only add tool_calls and function_call to res if they are set if message.get("tool_calls"): - serialized["tool_calls"] = message.get("tool_calls") - if message.get("function_call"): - serialized["function_call"] = message.get("function_call") + serialized["tool_calls"] = parse_tool_calls(message.get("tool_calls")) return serialized @@ -62,14 +76,16 @@ class LunaryLogger: version = importlib.metadata.version("lunary") # if version < 0.1.43 then raise ImportError if packaging.version.Version(version) < packaging.version.Version("0.1.43"): - print( + print( # noqa "Lunary version outdated. Required: >= 0.1.43. Upgrade via 'pip install lunary --upgrade'" ) raise ImportError self.lunary_client = lunary except ImportError: - print("Lunary not installed. Please install it using 'pip install lunary'") + print( # noqa + "Lunary not installed. Please install it using 'pip install lunary'" + ) # noqa raise ImportError def log_event( @@ -93,8 +109,13 @@ class LunaryLogger: print_verbose(f"Lunary Logging - Logging request for model {model}") litellm_params = kwargs.get("litellm_params", {}) + optional_params = kwargs.get("optional_params", {}) metadata = litellm_params.get("metadata", {}) or {} + if optional_params: + # merge into extra + extra = {**extra, **optional_params} + tags = litellm_params.pop("tags", None) or [] if extra: @@ -104,7 +125,7 @@ class LunaryLogger: # keep only serializable types for param, value in extra.items(): - if not isinstance(value, (str, int, bool, float)): + if not isinstance(value, (str, int, bool, float)) and param != "tools": try: extra[param] = str(value) except: @@ -140,7 +161,7 @@ class LunaryLogger: metadata=metadata, runtime="litellm", tags=tags, - extra=extra, + params=extra, ) self.lunary_client.track_event( diff --git a/litellm/integrations/openmeter.py b/litellm/integrations/openmeter.py index 248b83f4d..2c470d6f4 100644 --- a/litellm/integrations/openmeter.py +++ b/litellm/integrations/openmeter.py @@ -2,10 +2,7 @@ ## On Success events log cost to OpenMeter - https://github.com/BerriAI/litellm/issues/1268 import dotenv, os, json -import requests import litellm - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -60,12 +57,16 @@ class OpenMeterLogger(CustomLogger): "total_tokens": response_obj["usage"].get("total_tokens"), } + subject = (kwargs.get("user", None),) # end-user passed in via 'user' param + if not subject: + raise Exception("OpenMeter: user is required") + return { "specversion": "1.0", "type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"), "id": call_id, "time": dt, - "subject": kwargs.get("user", ""), # end-user passed in via 'user' param + "subject": subject, "source": "litellm-proxy", "data": {"model": model, "cost": cost, **usage}, } @@ -80,14 +81,23 @@ class OpenMeterLogger(CustomLogger): api_key = os.getenv("OPENMETER_API_KEY") _data = self._common_logic(kwargs=kwargs, response_obj=response_obj) - self.sync_http_handler.post( - url=_url, - data=_data, - headers={ - "Content-Type": "application/cloudevents+json", - "Authorization": "Bearer {}".format(api_key), - }, - ) + _headers = { + "Content-Type": "application/cloudevents+json", + "Authorization": "Bearer {}".format(api_key), + } + + try: + response = self.sync_http_handler.post( + url=_url, + data=json.dumps(_data), + headers=_headers, + ) + + response.raise_for_status() + except Exception as e: + if hasattr(response, "text"): + litellm.print_verbose(f"\nError Message: {response.text}") + raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): _url = os.getenv("OPENMETER_API_ENDPOINT", "https://openmeter.cloud") diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index e3c6e8e77..6fbc6ca4c 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -3,9 +3,7 @@ # On success, log events to Prometheus import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv +import requests # type: ignore import traceback import datetime, subprocess, sys import litellm, uuid @@ -19,7 +17,6 @@ class PrometheusLogger: **kwargs, ): try: - print(f"in init prometheus metrics") from prometheus_client import Counter self.litellm_llm_api_failed_requests_metric = Counter( diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 0249a71d0..8fce8930d 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -4,9 +4,7 @@ import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv +import requests # type: ignore import traceback import datetime, subprocess, sys import litellm, uuid @@ -183,7 +181,6 @@ class PrometheusServicesLogger: ) async def async_service_failure_hook(self, payload: ServiceLoggerPayload): - print(f"received error payload: {payload.error}") if self.mock_testing: self.mock_testing_failure_calls += 1 diff --git a/litellm/integrations/prompt_layer.py b/litellm/integrations/prompt_layer.py index 39a80940b..531ed75fe 100644 --- a/litellm/integrations/prompt_layer.py +++ b/litellm/integrations/prompt_layer.py @@ -1,12 +1,11 @@ #### What this does #### # On success, logs events to Promptlayer import dotenv, os -import requests +import requests # type: ignore from pydantic import BaseModel - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback + class PromptLayerLogger: # Class variables or attributes def __init__(self): @@ -32,7 +31,11 @@ class PromptLayerLogger: tags = kwargs["litellm_params"]["metadata"]["pl_tags"] # Remove "pl_tags" from metadata - metadata = {k:v for k, v in kwargs["litellm_params"]["metadata"].items() if k != "pl_tags"} + metadata = { + k: v + for k, v in kwargs["litellm_params"]["metadata"].items() + if k != "pl_tags" + } print_verbose( f"Prompt Layer Logging - Enters logging function for model kwargs: {new_kwargs}\n, response: {response_obj}" diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index dc35430bc..d131e44f0 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -1,10 +1,7 @@ #### What this does #### # On success + failure, log events to Supabase -import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv +import os import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index a9aba2f1c..04e5a4d1b 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -1,35 +1,90 @@ #### What this does #### # Class for sending Slack Alerts # import dotenv, os - -dotenv.load_dotenv() # Loading env variables using dotenv -import copy -import traceback +from litellm.proxy._types import UserAPIKeyAuth from litellm._logging import verbose_logger, verbose_proxy_logger -import litellm +import litellm, threading from typing import List, Literal, Any, Union, Optional, Dict from litellm.caching import DualCache import asyncio import aiohttp from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler import datetime +from pydantic import BaseModel +from enum import Enum +from datetime import datetime as dt, timedelta +from litellm.integrations.custom_logger import CustomLogger +import random -class SlackAlerting: +class LiteLLMBase(BaseModel): + """ + Implements default functions, all pydantic objects should have. + """ + + def json(self, **kwargs): + try: + return self.model_dump() # noqa + except: + # if using pydantic v1 + return self.dict() + + +class SlackAlertingArgs(LiteLLMBase): + default_daily_report_frequency: int = 12 * 60 * 60 # 12 hours + daily_report_frequency: int = int(os.getenv("SLACK_DAILY_REPORT_FREQUENCY", default_daily_report_frequency)) + report_check_interval: int = 5 * 60 # 5 minutes + + +class DeploymentMetrics(LiteLLMBase): + """ + Metrics per deployment, stored in cache + + Used for daily reporting + """ + + id: str + """id of deployment in router model list""" + + failed_request: bool + """did it fail the request?""" + + latency_per_output_token: Optional[float] + """latency/output token of deployment""" + + updated_at: dt + """Current time of deployment being updated""" + + +class SlackAlertingCacheKeys(Enum): + """ + Enum for deployment daily metrics keys - {deployment_id}:{enum} + """ + + failed_requests_key = "failed_requests_daily_metrics" + latency_key = "latency_daily_metrics" + report_sent_key = "daily_metrics_report_sent" + + +class SlackAlerting(CustomLogger): + """ + Class for sending Slack Alerts + """ + # Class variables or attributes def __init__( self, - alerting_threshold: float = 300, + internal_usage_cache: Optional[DualCache] = None, + alerting_threshold: float = 300, # threshold for slow / hanging llm responses (in seconds) alerting: Optional[List] = [], - alert_types: Optional[ - List[ - Literal[ - "llm_exceptions", - "llm_too_slow", - "llm_requests_hanging", - "budget_alerts", - "db_exceptions", - ] + alert_types: List[ + Literal[ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + "daily_reports", ] ] = [ "llm_exceptions", @@ -37,18 +92,23 @@ class SlackAlerting: "llm_requests_hanging", "budget_alerts", "db_exceptions", + "daily_reports", ], alert_to_webhook_url: Optional[ Dict ] = None, # if user wants to separate alerts to diff channels + alerting_args={}, + default_webhook_url: Optional[str] = None, ): self.alerting_threshold = alerting_threshold self.alerting = alerting self.alert_types = alert_types - self.internal_usage_cache = DualCache() + self.internal_usage_cache = internal_usage_cache or DualCache() self.async_http_handler = AsyncHTTPHandler() self.alert_to_webhook_url = alert_to_webhook_url - pass + self.is_running = False + self.alerting_args = SlackAlertingArgs(**alerting_args) + self.default_webhook_url = default_webhook_url def update_values( self, @@ -56,6 +116,7 @@ class SlackAlerting: alerting_threshold: Optional[float] = None, alert_types: Optional[List] = None, alert_to_webhook_url: Optional[Dict] = None, + alerting_args: Optional[Dict] = None, ): if alerting is not None: self.alerting = alerting @@ -63,7 +124,8 @@ class SlackAlerting: self.alerting_threshold = alerting_threshold if alert_types is not None: self.alert_types = alert_types - + if alerting_args is not None: + self.alerting_args = SlackAlertingArgs(**alerting_args) if alert_to_webhook_url is not None: # update the dict if self.alert_to_webhook_url is None: @@ -90,18 +152,23 @@ class SlackAlerting: def _add_langfuse_trace_id_to_alert( self, - request_info: str, request_data: Optional[dict] = None, - kwargs: Optional[dict] = None, - type: Literal["hanging_request", "slow_response"] = "hanging_request", - start_time: Optional[datetime.datetime] = None, - end_time: Optional[datetime.datetime] = None, - ): + ) -> Optional[str]: + """ + Returns langfuse trace url + """ # do nothing for now - pass - return request_info + if ( + request_data is not None + and request_data.get("metadata", {}).get("trace_id", None) is not None + ): + trace_id = request_data["metadata"]["trace_id"] + if litellm.utils.langFuseLogger is not None: + base_url = litellm.utils.langFuseLogger.Langfuse.base_url + return f"{base_url}/trace/{trace_id}" + return None - def _response_taking_too_long_callback( + def _response_taking_too_long_callback_helper( self, kwargs, # kwargs to completion start_time, @@ -166,12 +233,14 @@ class SlackAlerting: return time_difference_float, model, api_base, messages = ( - self._response_taking_too_long_callback( + self._response_taking_too_long_callback_helper( kwargs=kwargs, start_time=start_time, end_time=end_time, ) ) + if litellm.turn_off_message_logging: + messages = "Message not logged. `litellm.turn_off_message_logging=True`." request_info = f"\nRequest Model: `{model}`\nAPI Base: `{api_base}`\nMessages: `{messages}`" slow_message = f"`Responses are slow - {round(time_difference_float,2)}s response time > Alerting threshold: {self.alerting_threshold}s`" if time_difference_float > self.alerting_threshold: @@ -182,6 +251,9 @@ class SlackAlerting: and "metadata" in kwargs["litellm_params"] ): _metadata = kwargs["litellm_params"]["metadata"] + request_info = litellm.utils._add_key_name_and_team_to_alert( + request_info=request_info, metadata=_metadata + ) _deployment_latency_map = self._get_deployment_latencies_to_alert( metadata=_metadata @@ -196,8 +268,178 @@ class SlackAlerting: alert_type="llm_too_slow", ) - async def log_failure_event(self, original_exception: Exception): - pass + async def async_update_daily_reports( + self, deployment_metrics: DeploymentMetrics + ) -> int: + """ + Store the perf by deployment in cache + - Number of failed requests per deployment + - Latency / output tokens per deployment + + 'deployment_id:daily_metrics:failed_requests' + 'deployment_id:daily_metrics:latency_per_output_token' + + Returns + int - count of metrics set (1 - if just latency, 2 - if failed + latency) + """ + + return_val = 0 + try: + ## FAILED REQUESTS ## + if deployment_metrics.failed_request: + await self.internal_usage_cache.async_increment_cache( + key="{}:{}".format( + deployment_metrics.id, + SlackAlertingCacheKeys.failed_requests_key.value, + ), + value=1, + ) + + return_val += 1 + + ## LATENCY ## + if deployment_metrics.latency_per_output_token is not None: + await self.internal_usage_cache.async_increment_cache( + key="{}:{}".format( + deployment_metrics.id, SlackAlertingCacheKeys.latency_key.value + ), + value=deployment_metrics.latency_per_output_token, + ) + + return_val += 1 + + return return_val + except Exception as e: + return 0 + + async def send_daily_reports(self, router) -> bool: + """ + Send a daily report on: + - Top 5 deployments with most failed requests + - Top 5 slowest deployments (normalized by latency/output tokens) + + Get the value from redis cache (if available) or in-memory and send it + + Cleanup: + - reset values in cache -> prevent memory leak + + Returns: + True -> if successfuly sent + False -> if not sent + """ + + ids = router.get_model_ids() + + # get keys + failed_request_keys = [ + "{}:{}".format(id, SlackAlertingCacheKeys.failed_requests_key.value) + for id in ids + ] + latency_keys = [ + "{}:{}".format(id, SlackAlertingCacheKeys.latency_key.value) for id in ids + ] + + combined_metrics_keys = failed_request_keys + latency_keys # reduce cache calls + + combined_metrics_values = await self.internal_usage_cache.async_batch_get_cache( + keys=combined_metrics_keys + ) # [1, 2, None, ..] + + all_none = True + for val in combined_metrics_values: + if val is not None: + all_none = False + + if all_none: + return False + + failed_request_values = combined_metrics_values[ + : len(failed_request_keys) + ] # # [1, 2, None, ..] + latency_values = combined_metrics_values[len(failed_request_keys) :] + + # find top 5 failed + ## Replace None values with a placeholder value (-1 in this case) + placeholder_value = 0 + replaced_failed_values = [ + value if value is not None else placeholder_value + for value in failed_request_values + ] + + ## Get the indices of top 5 keys with the highest numerical values (ignoring None values) + top_5_failed = sorted( + range(len(replaced_failed_values)), + key=lambda i: replaced_failed_values[i], + reverse=True, + )[:5] + + # find top 5 slowest + # Replace None values with a placeholder value (-1 in this case) + placeholder_value = 0 + replaced_slowest_values = [ + value if value is not None else placeholder_value + for value in latency_values + ] + + # Get the indices of top 5 values with the highest numerical values (ignoring None values) + top_5_slowest = sorted( + range(len(replaced_slowest_values)), + key=lambda i: replaced_slowest_values[i], + reverse=True, + )[:5] + + # format alert -> return the litellm model name + api base + message = f"\n\nHere are today's key metrics 📈: \n\n" + + message += "\n\n*❗️ Top 5 Deployments with Most Failed Requests:*\n\n" + for i in range(len(top_5_failed)): + key = failed_request_keys[top_5_failed[i]].split(":")[0] + _deployment = router.get_model_info(key) + if isinstance(_deployment, dict): + deployment_name = _deployment["litellm_params"].get("model", "") + else: + return False + + api_base = litellm.get_api_base( + model=deployment_name, + optional_params=( + _deployment["litellm_params"] if _deployment is not None else {} + ), + ) + if api_base is None: + api_base = "" + value = replaced_failed_values[top_5_failed[i]] + message += f"\t{i+1}. Deployment: `{deployment_name}`, Failed Requests: `{value}`, API Base: `{api_base}`\n" + + message += "\n\n*😅 Top 5 Slowest Deployments:*\n\n" + for i in range(len(top_5_slowest)): + key = latency_keys[top_5_slowest[i]].split(":")[0] + _deployment = router.get_model_info(key) + if _deployment is not None: + deployment_name = _deployment["litellm_params"].get("model", "") + else: + deployment_name = "" + api_base = litellm.get_api_base( + model=deployment_name, + optional_params=( + _deployment["litellm_params"] if _deployment is not None else {} + ), + ) + value = round(replaced_slowest_values[top_5_slowest[i]], 3) + message += f"\t{i+1}. Deployment: `{deployment_name}`, Latency per output token: `{value}s/token`, API Base: `{api_base}`\n\n" + + # cache cleanup -> reset values to 0 + latency_cache_keys = [(key, 0) for key in latency_keys] + failed_request_cache_keys = [(key, 0) for key in failed_request_keys] + combined_metrics_cache_keys = latency_cache_keys + failed_request_cache_keys + await self.internal_usage_cache.async_batch_set_cache( + cache_list=combined_metrics_cache_keys + ) + + # send alert + await self.send_alert(message=message, level="Low", alert_type="daily_reports") + + return True async def response_taking_too_long( self, @@ -221,6 +463,11 @@ class SlackAlerting: messages = messages[:100] except: messages = "" + + if litellm.turn_off_message_logging: + messages = ( + "Message not logged. `litellm.turn_off_message_logging=True`." + ) request_info = f"\nRequest Model: `{model}`\nMessages: `{messages}`" else: request_info = "" @@ -255,6 +502,11 @@ class SlackAlerting: # in that case we fallback to the api base set in the request metadata _metadata = request_data["metadata"] _api_base = _metadata.get("api_base", "") + + request_info = litellm.utils._add_key_name_and_team_to_alert( + request_info=request_info, metadata=_metadata + ) + if _api_base is None: _api_base = "" request_info += f"\nAPI Base: `{_api_base}`" @@ -264,14 +516,13 @@ class SlackAlerting: ) if "langfuse" in litellm.success_callback: - request_info = self._add_langfuse_trace_id_to_alert( - request_info=request_info, + langfuse_url = self._add_langfuse_trace_id_to_alert( request_data=request_data, - type="hanging_request", - start_time=start_time, - end_time=end_time, ) + if langfuse_url is not None: + request_info += "\n🪢 Langfuse Trace: {}".format(langfuse_url) + # add deployment latencies to alert _deployment_latency_map = self._get_deployment_latencies_to_alert( metadata=request_data.get("metadata", {}) @@ -404,6 +655,53 @@ class SlackAlerting: return + async def model_added_alert(self, model_name: str, litellm_model_name: str): + model_info = litellm.model_cost.get(litellm_model_name, {}) + model_info_str = "" + for k, v in model_info.items(): + if k == "input_cost_per_token" or k == "output_cost_per_token": + # when converting to string it should not be 1.63e-06 + v = "{:.8f}".format(v) + + model_info_str += f"{k}: {v}\n" + + message = f""" +*🚅 New Model Added* +Model Name: `{model_name}` + +Usage OpenAI Python SDK: +``` +import openai +client = openai.OpenAI( + api_key="your_api_key", + base_url={os.getenv("PROXY_BASE_URL", "http://0.0.0.0:4000")} +) + +response = client.chat.completions.create( + model="{model_name}", # model to send to the proxy + messages = [ + {{ + "role": "user", + "content": "this is a test request, write a short poem" + }} + ] +) +``` + +Model Info: +``` +{model_info_str} +``` +""" + + await self.send_alert( + message=message, level="Low", alert_type="new_model_added" + ) + pass + + async def model_removed_alert(self, model_name: str): + pass + async def send_alert( self, message: str, @@ -414,7 +712,11 @@ class SlackAlerting: "llm_requests_hanging", "budget_alerts", "db_exceptions", + "daily_reports", + "new_model_added", + "cooldown_deployment", ], + **kwargs, ): """ Alerting based on thresholds: - https://github.com/BerriAI/litellm/issues/1298 @@ -439,9 +741,16 @@ class SlackAlerting: # Get the current timestamp current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) - formatted_message = ( - f"Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" - ) + if alert_type == "daily_reports" or alert_type == "new_model_added": + formatted_message = message + else: + formatted_message = ( + f"Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}" + ) + + if kwargs: + for key, value in kwargs.items(): + formatted_message += f"\n\n{key}: `{value}`\n\n" if _proxy_base_url is not None: formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`" @@ -451,6 +760,8 @@ class SlackAlerting: and alert_type in self.alert_to_webhook_url ): slack_webhook_url = self.alert_to_webhook_url[alert_type] + elif self.default_webhook_url is not None: + slack_webhook_url = self.default_webhook_url else: slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None) @@ -468,3 +779,201 @@ class SlackAlerting: pass else: print("Error sending slack alert. Error=", response.text) # noqa + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + """Log deployment latency""" + if "daily_reports" in self.alert_types: + model_id = ( + kwargs.get("litellm_params", {}).get("model_info", {}).get("id", "") + ) + response_s: timedelta = end_time - start_time + + final_value = response_s + total_tokens = 0 + + if isinstance(response_obj, litellm.ModelResponse): + completion_tokens = response_obj.usage.completion_tokens + final_value = float(response_s.total_seconds() / completion_tokens) + + await self.async_update_daily_reports( + DeploymentMetrics( + id=model_id, + failed_request=False, + latency_per_output_token=final_value, + updated_at=litellm.utils.get_utc_datetime(), + ) + ) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + """Log failure + deployment latency""" + if "daily_reports" in self.alert_types: + model_id = ( + kwargs.get("litellm_params", {}).get("model_info", {}).get("id", "") + ) + await self.async_update_daily_reports( + DeploymentMetrics( + id=model_id, + failed_request=True, + latency_per_output_token=None, + updated_at=litellm.utils.get_utc_datetime(), + ) + ) + + async def _run_scheduler_helper(self, llm_router) -> bool: + """ + Returns: + - True -> report sent + - False -> report not sent + """ + report_sent_bool = False + + report_sent = await self.internal_usage_cache.async_get_cache( + key=SlackAlertingCacheKeys.report_sent_key.value + ) # None | datetime + + current_time = litellm.utils.get_utc_datetime() + + if report_sent is None: + _current_time = current_time.isoformat() + await self.internal_usage_cache.async_set_cache( + key=SlackAlertingCacheKeys.report_sent_key.value, + value=_current_time, + ) + else: + # check if current time - interval >= time last sent + delta = current_time - timedelta( + seconds=self.alerting_args.daily_report_frequency + ) + + if isinstance(report_sent, str): + report_sent = dt.fromisoformat(report_sent) + + if delta >= report_sent: + # Sneak in the reporting logic here + await self.send_daily_reports(router=llm_router) + # Also, don't forget to update the report_sent time after sending the report! + _current_time = current_time.isoformat() + await self.internal_usage_cache.async_set_cache( + key=SlackAlertingCacheKeys.report_sent_key.value, + value=_current_time, + ) + report_sent_bool = True + + return report_sent_bool + + async def _run_scheduled_daily_report(self, llm_router: Optional[Any] = None): + """ + If 'daily_reports' enabled + + Ping redis cache every 5 minutes to check if we should send the report + + If yes -> call send_daily_report() + """ + if llm_router is None or self.alert_types is None: + return + + if "daily_reports" in self.alert_types: + while True: + await self._run_scheduler_helper(llm_router=llm_router) + interval = random.randint( + self.alerting_args.report_check_interval - 3, + self.alerting_args.report_check_interval + 3, + ) # shuffle to prevent collisions + await asyncio.sleep(interval) + return + + async def send_weekly_spend_report(self): + """ """ + try: + from litellm.proxy.proxy_server import _get_spend_report_for_time_range + + todays_date = datetime.datetime.now().date() + week_before = todays_date - datetime.timedelta(days=7) + + weekly_spend_per_team, weekly_spend_per_tag = ( + await _get_spend_report_for_time_range( + start_date=week_before.strftime("%Y-%m-%d"), + end_date=todays_date.strftime("%Y-%m-%d"), + ) + ) + + _weekly_spend_message = f"*💸 Weekly Spend Report for `{week_before.strftime('%m-%d-%Y')} - {todays_date.strftime('%m-%d-%Y')}` *\n" + + if weekly_spend_per_team is not None: + _weekly_spend_message += "\n*Team Spend Report:*\n" + for spend in weekly_spend_per_team: + _team_spend = spend["total_spend"] + _team_spend = float(_team_spend) + # round to 4 decimal places + _team_spend = round(_team_spend, 4) + _weekly_spend_message += ( + f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n" + ) + + if weekly_spend_per_tag is not None: + _weekly_spend_message += "\n*Tag Spend Report:*\n" + for spend in weekly_spend_per_tag: + _tag_spend = spend["total_spend"] + _tag_spend = float(_tag_spend) + # round to 4 decimal places + _tag_spend = round(_tag_spend, 4) + _weekly_spend_message += f"Tag: `{spend['individual_request_tag']}` | Spend: `${_tag_spend}`\n" + + await self.send_alert( + message=_weekly_spend_message, + level="Low", + alert_type="daily_reports", + ) + except Exception as e: + verbose_proxy_logger.error("Error sending weekly spend report", e) + + async def send_monthly_spend_report(self): + """ """ + try: + from calendar import monthrange + + from litellm.proxy.proxy_server import _get_spend_report_for_time_range + + todays_date = datetime.datetime.now().date() + first_day_of_month = todays_date.replace(day=1) + _, last_day_of_month = monthrange(todays_date.year, todays_date.month) + last_day_of_month = first_day_of_month + datetime.timedelta( + days=last_day_of_month - 1 + ) + + monthly_spend_per_team, monthly_spend_per_tag = ( + await _get_spend_report_for_time_range( + start_date=first_day_of_month.strftime("%Y-%m-%d"), + end_date=last_day_of_month.strftime("%Y-%m-%d"), + ) + ) + + _spend_message = f"*💸 Monthly Spend Report for `{first_day_of_month.strftime('%m-%d-%Y')} - {last_day_of_month.strftime('%m-%d-%Y')}` *\n" + + if monthly_spend_per_team is not None: + _spend_message += "\n*Team Spend Report:*\n" + for spend in monthly_spend_per_team: + _team_spend = spend["total_spend"] + _team_spend = float(_team_spend) + # round to 4 decimal places + _team_spend = round(_team_spend, 4) + _spend_message += ( + f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n" + ) + + if monthly_spend_per_tag is not None: + _spend_message += "\n*Tag Spend Report:*\n" + for spend in monthly_spend_per_tag: + _tag_spend = spend["total_spend"] + _tag_spend = float(_tag_spend) + # round to 4 decimal places + _tag_spend = round(_tag_spend, 4) + _spend_message += f"Tag: `{spend['individual_request_tag']}` | Spend: `${_tag_spend}`\n" + + await self.send_alert( + message=_spend_message, + level="Low", + alert_type="daily_reports", + ) + except Exception as e: + verbose_proxy_logger.error("Error sending weekly spend report", e) diff --git a/litellm/integrations/supabase.py b/litellm/integrations/supabase.py index a99e4abc4..4e6bf517f 100644 --- a/litellm/integrations/supabase.py +++ b/litellm/integrations/supabase.py @@ -2,9 +2,7 @@ # On success + failure, log events to Supabase import dotenv, os -import requests - -dotenv.load_dotenv() # Loading env variables using dotenv +import requests # type: ignore import traceback import datetime, subprocess, sys import litellm diff --git a/litellm/integrations/weights_biases.py b/litellm/integrations/weights_biases.py index 53e6070a5..a56233b22 100644 --- a/litellm/integrations/weights_biases.py +++ b/litellm/integrations/weights_biases.py @@ -21,11 +21,11 @@ try: # contains a (known) object attribute object: Literal["chat.completion", "edit", "text_completion"] - def __getitem__(self, key: K) -> V: - ... # pragma: no cover + def __getitem__(self, key: K) -> V: ... # noqa - def get(self, key: K, default: Optional[V] = None) -> Optional[V]: - ... # pragma: no cover + def get( # noqa + self, key: K, default: Optional[V] = None + ) -> Optional[V]: ... # pragma: no cover class OpenAIRequestResponseResolver: def __call__( @@ -173,12 +173,11 @@ except: #### What this does #### # On success, logs events to Langfuse -import dotenv, os +import os import requests import requests from datetime import datetime -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/llms/ai21.py b/litellm/llms/ai21.py index 73d5afebe..a39a83f15 100644 --- a/litellm/llms/ai21.py +++ b/litellm/llms/ai21.py @@ -1,8 +1,8 @@ import os, types, traceback import json from enum import Enum -import requests -import time, httpx +import requests # type: ignore +import time, httpx # type: ignore from typing import Callable, Optional from litellm.utils import ModelResponse, Choices, Message import litellm diff --git a/litellm/llms/aleph_alpha.py b/litellm/llms/aleph_alpha.py index 86a30a9ec..7edd11964 100644 --- a/litellm/llms/aleph_alpha.py +++ b/litellm/llms/aleph_alpha.py @@ -1,12 +1,12 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional import litellm from litellm.utils import ModelResponse, Choices, Message, Usage -import httpx +import httpx # type: ignore class AlephAlphaError(Exception): diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 24d889b0f..97a473a2e 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -1,15 +1,15 @@ import os, types import json from enum import Enum -import requests, copy +import requests, copy # type: ignore import time -from typing import Callable, Optional, List +from typing import Callable, Optional, List, Union from litellm.utils import ModelResponse, Usage, map_finish_reason, CustomStreamWrapper import litellm from .prompt_templates.factory import prompt_factory, custom_prompt from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from .base import BaseLLM -import httpx +import httpx # type: ignore class AnthropicConstants(Enum): @@ -84,6 +84,51 @@ class AnthropicConfig: and v is not None } + def get_supported_openai_params(self): + return [ + "stream", + "stop", + "temperature", + "top_p", + "max_tokens", + "tools", + "tool_choice", + ] + + def map_openai_params(self, non_default_params: dict, optional_params: dict): + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["max_tokens"] = value + if param == "tools": + optional_params["tools"] = value + if param == "stream" and value == True: + optional_params["stream"] = value + if param == "stop": + if isinstance(value, str): + if ( + value == "\n" + ) and litellm.drop_params == True: # anthropic doesn't allow whitespace characters as stop-sequences + continue + value = [value] + elif isinstance(value, list): + new_v = [] + for v in value: + if ( + v == "\n" + ) and litellm.drop_params == True: # anthropic doesn't allow whitespace characters as stop-sequences + continue + new_v.append(v) + if len(new_v) > 0: + value = new_v + else: + continue + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + return optional_params + # makes headers for API call def validate_environment(api_key, user_headers): @@ -106,19 +151,135 @@ class AnthropicChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() + def process_streaming_response( + self, + model: str, + response: Union[requests.Response, httpx.Response], + model_response: ModelResponse, + stream: bool, + logging_obj: litellm.utils.Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: List, + print_verbose, + encoding, + ) -> CustomStreamWrapper: + """ + Return stream object for tool-calling + streaming + """ + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + ## RESPONSE OBJECT + try: + completion_response = response.json() + except: + raise AnthropicError( + message=response.text, status_code=response.status_code + ) + text_content = "" + tool_calls = [] + for content in completion_response["content"]: + if content["type"] == "text": + text_content += content["text"] + ## TOOL CALLING + elif content["type"] == "tool_use": + tool_calls.append( + { + "id": content["id"], + "type": "function", + "function": { + "name": content["name"], + "arguments": json.dumps(content["input"]), + }, + } + ) + if "error" in completion_response: + raise AnthropicError( + message=str(completion_response["error"]), + status_code=response.status_code, + ) + _message = litellm.Message( + tool_calls=tool_calls, + content=text_content or None, + ) + model_response.choices[0].message = _message # type: ignore + model_response._hidden_params["original_response"] = completion_response[ + "content" + ] # allow user to access raw anthropic tool calling response + + model_response.choices[0].finish_reason = map_finish_reason( + completion_response["stop_reason"] + ) + + print_verbose("INSIDE ANTHROPIC STREAMING TOOL CALLING CONDITION BLOCK") + # return an iterator + streaming_model_response = ModelResponse(stream=True) + streaming_model_response.choices[0].finish_reason = model_response.choices[ # type: ignore + 0 + ].finish_reason + # streaming_model_response.choices = [litellm.utils.StreamingChoices()] + streaming_choice = litellm.utils.StreamingChoices() + streaming_choice.index = model_response.choices[0].index + _tool_calls = [] + print_verbose( + f"type of model_response.choices[0]: {type(model_response.choices[0])}" + ) + print_verbose(f"type of streaming_choice: {type(streaming_choice)}") + if isinstance(model_response.choices[0], litellm.Choices): + if getattr( + model_response.choices[0].message, "tool_calls", None + ) is not None and isinstance( + model_response.choices[0].message.tool_calls, list + ): + for tool_call in model_response.choices[0].message.tool_calls: + _tool_call = {**tool_call.dict(), "index": 0} + _tool_calls.append(_tool_call) + delta_obj = litellm.utils.Delta( + content=getattr(model_response.choices[0].message, "content", None), + role=model_response.choices[0].message.role, + tool_calls=_tool_calls, + ) + streaming_choice.delta = delta_obj + streaming_model_response.choices = [streaming_choice] + completion_stream = ModelResponseIterator( + model_response=streaming_model_response + ) + print_verbose( + "Returns anthropic CustomStreamWrapper with 'cached_response' streaming object" + ) + return CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="cached_response", + logging_obj=logging_obj, + ) + else: + raise AnthropicError( + status_code=422, + message="Unprocessable response object - {}".format(response.text), + ) + def process_response( self, - model, - response, - model_response, - _is_function_call, - stream, - logging_obj, - api_key, - data, - messages, + model: str, + response: Union[requests.Response, httpx.Response], + model_response: ModelResponse, + stream: bool, + logging_obj: litellm.utils.Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: List, print_verbose, - ): + encoding, + ) -> ModelResponse: ## LOGGING logging_obj.post_call( input=messages, @@ -139,11 +300,6 @@ class AnthropicChatCompletion(BaseLLM): message=str(completion_response["error"]), status_code=response.status_code, ) - elif len(completion_response["content"]) == 0: - raise AnthropicError( - message="No content in response", - status_code=response.status_code, - ) else: text_content = "" tool_calls = [] @@ -176,51 +332,6 @@ class AnthropicChatCompletion(BaseLLM): completion_response["stop_reason"] ) - print_verbose(f"_is_function_call: {_is_function_call}; stream: {stream}") - if _is_function_call and stream: - print_verbose("INSIDE ANTHROPIC STREAMING TOOL CALLING CONDITION BLOCK") - # return an iterator - streaming_model_response = ModelResponse(stream=True) - streaming_model_response.choices[0].finish_reason = model_response.choices[ - 0 - ].finish_reason - # streaming_model_response.choices = [litellm.utils.StreamingChoices()] - streaming_choice = litellm.utils.StreamingChoices() - streaming_choice.index = model_response.choices[0].index - _tool_calls = [] - print_verbose( - f"type of model_response.choices[0]: {type(model_response.choices[0])}" - ) - print_verbose(f"type of streaming_choice: {type(streaming_choice)}") - if isinstance(model_response.choices[0], litellm.Choices): - if getattr( - model_response.choices[0].message, "tool_calls", None - ) is not None and isinstance( - model_response.choices[0].message.tool_calls, list - ): - for tool_call in model_response.choices[0].message.tool_calls: - _tool_call = {**tool_call.dict(), "index": 0} - _tool_calls.append(_tool_call) - delta_obj = litellm.utils.Delta( - content=getattr(model_response.choices[0].message, "content", None), - role=model_response.choices[0].message.role, - tool_calls=_tool_calls, - ) - streaming_choice.delta = delta_obj - streaming_model_response.choices = [streaming_choice] - completion_stream = ModelResponseIterator( - model_response=streaming_model_response - ) - print_verbose( - "Returns anthropic CustomStreamWrapper with 'cached_response' streaming object" - ) - return CustomStreamWrapper( - completion_stream=completion_stream, - model=model, - custom_llm_provider="cached_response", - logging_obj=logging_obj, - ) - ## CALCULATING USAGE prompt_tokens = completion_response["usage"]["input_tokens"] completion_tokens = completion_response["usage"]["output_tokens"] @@ -233,7 +344,7 @@ class AnthropicChatCompletion(BaseLLM): completion_tokens=completion_tokens, total_tokens=total_tokens, ) - model_response.usage = usage + setattr(model_response, "usage", usage) # type: ignore return model_response async def acompletion_stream_function( @@ -249,7 +360,7 @@ class AnthropicChatCompletion(BaseLLM): logging_obj, stream, _is_function_call, - data=None, + data: dict, optional_params=None, litellm_params=None, logger_fn=None, @@ -291,29 +402,44 @@ class AnthropicChatCompletion(BaseLLM): logging_obj, stream, _is_function_call, - data=None, - optional_params=None, + data: dict, + optional_params: dict, litellm_params=None, logger_fn=None, headers={}, - ): + ) -> Union[ModelResponse, CustomStreamWrapper]: self.async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) response = await self.async_handler.post( api_base, headers=headers, data=json.dumps(data) ) + if stream and _is_function_call: + return self.process_streaming_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, + ) return self.process_response( model=model, response=response, model_response=model_response, - _is_function_call=_is_function_call, stream=stream, logging_obj=logging_obj, api_key=api_key, data=data, messages=messages, print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, ) def completion( @@ -327,7 +453,7 @@ class AnthropicChatCompletion(BaseLLM): encoding, api_key, logging_obj, - optional_params=None, + optional_params: dict, acompletion=None, litellm_params=None, logger_fn=None, @@ -486,17 +612,33 @@ class AnthropicChatCompletion(BaseLLM): raise AnthropicError( status_code=response.status_code, message=response.text ) + + if stream and _is_function_call: + return self.process_streaming_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, + ) return self.process_response( model=model, response=response, model_response=model_response, - _is_function_call=_is_function_call, stream=stream, logging_obj=logging_obj, api_key=api_key, data=data, messages=messages, print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, ) def embedding(self): diff --git a/litellm/llms/anthropic_text.py b/litellm/llms/anthropic_text.py index cef31c269..0093d9f35 100644 --- a/litellm/llms/anthropic_text.py +++ b/litellm/llms/anthropic_text.py @@ -100,7 +100,7 @@ class AnthropicTextCompletion(BaseLLM): def __init__(self) -> None: super().__init__() - def process_response( + def _process_response( self, model_response: ModelResponse, response, encoding, prompt: str, model: str ): ## RESPONSE OBJECT @@ -171,7 +171,7 @@ class AnthropicTextCompletion(BaseLLM): additional_args={"complete_input_dict": data}, ) - response = self.process_response( + response = self._process_response( model_response=model_response, response=response, encoding=encoding, @@ -330,7 +330,7 @@ class AnthropicTextCompletion(BaseLLM): ) print_verbose(f"raw model_response: {response.text}") - response = self.process_response( + response = self._process_response( model_response=model_response, response=response, encoding=encoding, diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index 0fe5c4e7e..02fe4a08f 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -1,4 +1,4 @@ -from typing import Optional, Union, Any +from typing import Optional, Union, Any, Literal import types, requests from .base import BaseLLM from litellm.utils import ( @@ -8,14 +8,16 @@ from litellm.utils import ( CustomStreamWrapper, convert_to_model_response_object, TranscriptionResponse, + get_secret, ) -from typing import Callable, Optional, BinaryIO +from typing import Callable, Optional, BinaryIO, List from litellm import OpenAIConfig import litellm, json -import httpx +import httpx # type: ignore from .custom_httpx.azure_dall_e_2 import CustomHTTPTransport, AsyncCustomHTTPTransport from openai import AzureOpenAI, AsyncAzureOpenAI import uuid +import os class AzureOpenAIError(Exception): @@ -105,6 +107,12 @@ class AzureOpenAIConfig(OpenAIConfig): optional_params["azure_ad_token"] = value return optional_params + def get_eu_regions(self) -> List[str]: + """ + Source: https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-model-availability + """ + return ["europe", "sweden", "switzerland", "france", "uk"] + def select_azure_base_url_or_endpoint(azure_client_params: dict): # azure_client_params = { @@ -126,6 +134,51 @@ def select_azure_base_url_or_endpoint(azure_client_params: dict): return azure_client_params +def get_azure_ad_token_from_oidc(azure_ad_token: str): + azure_client_id = os.getenv("AZURE_CLIENT_ID", None) + azure_tenant = os.getenv("AZURE_TENANT_ID", None) + + if azure_client_id is None or azure_tenant is None: + raise AzureOpenAIError( + status_code=422, + message="AZURE_CLIENT_ID and AZURE_TENANT_ID must be set", + ) + + oidc_token = get_secret(azure_ad_token) + + if oidc_token is None: + raise AzureOpenAIError( + status_code=401, + message="OIDC token could not be retrieved from secret manager.", + ) + + req_token = httpx.post( + f"https://login.microsoftonline.com/{azure_tenant}/oauth2/v2.0/token", + data={ + "client_id": azure_client_id, + "grant_type": "client_credentials", + "scope": "https://cognitiveservices.azure.com/.default", + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) + + if req_token.status_code != 200: + raise AzureOpenAIError( + status_code=req_token.status_code, + message=req_token.text, + ) + + possible_azure_ad_token = req_token.json().get("access_token", None) + + if possible_azure_ad_token is None: + raise AzureOpenAIError( + status_code=422, message="Azure AD Token not returned" + ) + + return possible_azure_ad_token + + class AzureChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() @@ -137,6 +190,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: headers["api-key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) headers["Authorization"] = f"Bearer {azure_ad_token}" return headers @@ -151,7 +206,7 @@ class AzureChatCompletion(BaseLLM): api_type: str, azure_ad_token: str, print_verbose: Callable, - timeout, + timeout: Union[float, httpx.Timeout], logging_obj, optional_params, litellm_params, @@ -189,6 +244,9 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) + azure_client_params["azure_ad_token"] = azure_ad_token if acompletion is True: @@ -276,6 +334,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AzureOpenAI(**azure_client_params) @@ -351,6 +411,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token # setting Azure client @@ -422,6 +484,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AzureOpenAI(**azure_client_params) @@ -478,6 +542,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AsyncAzureOpenAI(**azure_client_params) @@ -599,6 +665,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token ## LOGGING @@ -755,6 +823,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if aimg_generation == True: @@ -833,6 +903,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if max_retries is not None: @@ -952,6 +1024,81 @@ class AzureChatCompletion(BaseLLM): ) raise e + def get_headers( + self, + model: Optional[str], + api_key: str, + api_base: str, + api_version: str, + timeout: float, + mode: str, + messages: Optional[list] = None, + input: Optional[list] = None, + prompt: Optional[str] = None, + ) -> dict: + client_session = litellm.client_session or httpx.Client( + transport=CustomHTTPTransport(), # handle dall-e-2 calls + ) + if "gateway.ai.cloudflare.com" in api_base: + ## build base url - assume api base includes resource name + if not api_base.endswith("/"): + api_base += "/" + api_base += f"{model}" + client = AzureOpenAI( + base_url=api_base, + api_version=api_version, + api_key=api_key, + timeout=timeout, + http_client=client_session, + ) + model = None + # cloudflare ai gateway, needs model=None + else: + client = AzureOpenAI( + api_version=api_version, + azure_endpoint=api_base, + api_key=api_key, + timeout=timeout, + http_client=client_session, + ) + + # only run this check if it's not cloudflare ai gateway + if model is None and mode != "image_generation": + raise Exception("model is not set") + + completion = None + + if messages is None: + messages = [{"role": "user", "content": "Hey"}] + try: + completion = client.chat.completions.with_raw_response.create( + model=model, # type: ignore + messages=messages, # type: ignore + ) + except Exception as e: + raise e + response = {} + + if completion is None or not hasattr(completion, "headers"): + raise Exception("invalid completion response") + + if ( + completion.headers.get("x-ratelimit-remaining-requests", None) is not None + ): # not provided for dall-e requests + response["x-ratelimit-remaining-requests"] = completion.headers[ + "x-ratelimit-remaining-requests" + ] + + if completion.headers.get("x-ratelimit-remaining-tokens", None) is not None: + response["x-ratelimit-remaining-tokens"] = completion.headers[ + "x-ratelimit-remaining-tokens" + ] + + if completion.headers.get("x-ms-region", None) is not None: + response["x-ms-region"] = completion.headers["x-ms-region"] + + return response + async def ahealth_check( self, model: Optional[str], @@ -963,7 +1110,7 @@ class AzureChatCompletion(BaseLLM): messages: Optional[list] = None, input: Optional[list] = None, prompt: Optional[str] = None, - ): + ) -> dict: client_session = litellm.aclient_session or httpx.AsyncClient( transport=AsyncCustomHTTPTransport(), # handle dall-e-2 calls ) @@ -1040,4 +1187,8 @@ class AzureChatCompletion(BaseLLM): response["x-ratelimit-remaining-tokens"] = completion.headers[ "x-ratelimit-remaining-tokens" ] + + if completion.headers.get("x-ms-region", None) is not None: + response["x-ms-region"] = completion.headers["x-ms-region"] + return response diff --git a/litellm/llms/azure_text.py b/litellm/llms/azure_text.py index e0d547477..640ab8222 100644 --- a/litellm/llms/azure_text.py +++ b/litellm/llms/azure_text.py @@ -1,5 +1,5 @@ from typing import Optional, Union, Any -import types, requests +import types, requests # type: ignore from .base import BaseLLM from litellm.utils import ( ModelResponse, diff --git a/litellm/llms/base.py b/litellm/llms/base.py index 62b8069f0..d940d9471 100644 --- a/litellm/llms/base.py +++ b/litellm/llms/base.py @@ -1,12 +1,32 @@ ## This is a template base class to be used for adding new LLM providers via API calls import litellm -import httpx -from typing import Optional +import httpx, requests +from typing import Optional, Union +from litellm.utils import Logging class BaseLLM: _client_session: Optional[httpx.Client] = None + def process_response( + self, + model: str, + response: Union[requests.Response, httpx.Response], + model_response: litellm.utils.ModelResponse, + stream: bool, + logging_obj: Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: list, + print_verbose, + encoding, + ) -> litellm.utils.ModelResponse: + """ + Helper function to process the response across sync + async completion calls + """ + return model_response + def create_client_session(self): if litellm.client_session: _client_session = litellm.client_session diff --git a/litellm/llms/baseten.py b/litellm/llms/baseten.py index 75db9ab46..643dae530 100644 --- a/litellm/llms/baseten.py +++ b/litellm/llms/baseten.py @@ -1,7 +1,7 @@ import os import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable from litellm.utils import ModelResponse, Usage diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 235c13c59..4314032e7 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -4,7 +4,13 @@ from enum import Enum import time, uuid from typing import Callable, Optional, Any, Union, List import litellm -from litellm.utils import ModelResponse, get_secret, Usage, ImageResponse +from litellm.utils import ( + ModelResponse, + get_secret, + Usage, + ImageResponse, + map_finish_reason, +) from .prompt_templates.factory import ( prompt_factory, custom_prompt, @@ -46,6 +52,16 @@ class AmazonBedrockGlobalConfig: optional_params[mapped_params[param]] = value return optional_params + def get_eu_regions(self) -> List[str]: + """ + Source: https://www.aws-services.info/bedrock.html + """ + return [ + "eu-west-1", + "eu-west-3", + "eu-central-1", + ] + class AmazonTitanConfig: """ @@ -157,6 +173,7 @@ class AmazonAnthropicClaude3Config: "stop", "temperature", "top_p", + "extra_headers", ] def map_openai_params(self, non_default_params: dict, optional_params: dict): @@ -524,6 +541,17 @@ class AmazonStabilityConfig: } +def add_custom_header(headers): + """Closure to capture the headers and add them.""" + + def callback(request, **kwargs): + """Actual callback function that Boto3 will call.""" + for header_name, header_value in headers.items(): + request.headers.add_header(header_name, header_value) + + return callback + + def init_bedrock_client( region_name=None, aws_access_key_id: Optional[str] = None, @@ -533,12 +561,13 @@ def init_bedrock_client( aws_session_name: Optional[str] = None, aws_profile_name: Optional[str] = None, aws_role_name: Optional[str] = None, - timeout: Optional[int] = None, + aws_web_identity_token: Optional[str] = None, + extra_headers: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, ): # check for custom AWS_REGION_NAME and use it if not passed to init_bedrock_client litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) standard_aws_region_name = get_secret("AWS_REGION", None) - ## CHECK IS 'os.environ/' passed in # Define the list of parameters to check params_to_check = [ @@ -549,6 +578,7 @@ def init_bedrock_client( aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ] # Iterate over parameters and update if needed @@ -564,6 +594,7 @@ def init_bedrock_client( aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ) = params_to_check ### SET REGION NAME @@ -592,10 +623,48 @@ def init_bedrock_client( import boto3 - config = boto3.session.Config(connect_timeout=timeout, read_timeout=timeout) + if isinstance(timeout, float): + config = boto3.session.Config(connect_timeout=timeout, read_timeout=timeout) + elif isinstance(timeout, httpx.Timeout): + config = boto3.session.Config( + connect_timeout=timeout.connect, read_timeout=timeout.read + ) + else: + config = boto3.session.Config() ### CHECK STS ### - if aws_role_name is not None and aws_session_name is not None: + if aws_web_identity_token is not None and aws_role_name is not None and aws_session_name is not None: + oidc_token = get_secret(aws_web_identity_token) + + if oidc_token is None: + raise BedrockError( + message="OIDC token could not be retrieved from secret manager.", + status_code=401, + ) + + sts_client = boto3.client( + "sts" + ) + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html + sts_response = sts_client.assume_role_with_web_identity( + RoleArn=aws_role_name, + RoleSessionName=aws_session_name, + WebIdentityToken=oidc_token, + DurationSeconds=3600, + ) + + client = boto3.client( + service_name="bedrock-runtime", + aws_access_key_id=sts_response["Credentials"]["AccessKeyId"], + aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"], + aws_session_token=sts_response["Credentials"]["SessionToken"], + region_name=region_name, + endpoint_url=endpoint_url, + config=config, + ) + elif aws_role_name is not None and aws_session_name is not None: # use sts if role name passed in sts_client = boto3.client( "sts", @@ -647,6 +716,10 @@ def init_bedrock_client( endpoint_url=endpoint_url, config=config, ) + if extra_headers: + client.meta.events.register( + "before-sign.bedrock-runtime.*", add_custom_header(extra_headers) + ) return client @@ -710,6 +783,7 @@ def completion( litellm_params=None, logger_fn=None, timeout=None, + extra_headers: Optional[dict] = None, ): exception_mapping_worked = False _is_function_call = False @@ -725,6 +799,7 @@ def completion( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = optional_params.pop("aws_bedrock_client", None) @@ -739,6 +814,8 @@ def completion( aws_role_name=aws_role_name, aws_session_name=aws_session_name, aws_profile_name=aws_profile_name, + aws_web_identity_token=aws_web_identity_token, + extra_headers=extra_headers, timeout=timeout, ) @@ -1043,7 +1120,9 @@ def completion( logging_obj=logging_obj, ) - model_response["finish_reason"] = response_body["stop_reason"] + model_response["finish_reason"] = map_finish_reason( + response_body["stop_reason"] + ) _usage = litellm.Usage( prompt_tokens=response_body["usage"]["input_tokens"], completion_tokens=response_body["usage"]["output_tokens"], @@ -1194,7 +1273,7 @@ def _embedding_func_single( "input_type", "search_document" ) # aws bedrock example default - https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/providers?model=cohere.embed-english-v3 data = {"texts": [input], **inference_params} # type: ignore - body = json.dumps(data).encode("utf-8") + body = json.dumps(data).encode("utf-8") # type: ignore ## LOGGING request_str = f""" response = client.invoke_model( @@ -1258,6 +1337,7 @@ def embedding( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = init_bedrock_client( @@ -1265,6 +1345,7 @@ def embedding( aws_secret_access_key=aws_secret_access_key, aws_region_name=aws_region_name, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_web_identity_token=aws_web_identity_token, aws_role_name=aws_role_name, aws_session_name=aws_session_name, ) @@ -1347,6 +1428,7 @@ def image_generation( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = init_bedrock_client( @@ -1354,6 +1436,7 @@ def image_generation( aws_secret_access_key=aws_secret_access_key, aws_region_name=aws_region_name, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_web_identity_token=aws_web_identity_token, aws_role_name=aws_role_name, aws_session_name=aws_session_name, timeout=timeout, @@ -1386,7 +1469,7 @@ def image_generation( ## LOGGING request_str = f""" response = client.invoke_model( - body={body}, + body={body}, # type: ignore modelId={modelId}, accept="application/json", contentType="application/json", diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py new file mode 100644 index 000000000..1ff3767bd --- /dev/null +++ b/litellm/llms/bedrock_httpx.py @@ -0,0 +1,733 @@ +# What is this? +## Initial implementation of calling bedrock via httpx client (allows for async calls). +## V0 - just covers cohere command-r support + +import os, types +import json +from enum import Enum +import requests, copy # type: ignore +import time +from typing import ( + Callable, + Optional, + List, + Literal, + Union, + Any, + TypedDict, + Tuple, + Iterator, + AsyncIterator, +) +from litellm.utils import ( + ModelResponse, + Usage, + map_finish_reason, + CustomStreamWrapper, + Message, + Choices, + get_secret, + Logging, +) +import litellm +from .prompt_templates.factory import prompt_factory, custom_prompt, cohere_message_pt +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from .base import BaseLLM +import httpx # type: ignore +from .bedrock import BedrockError, convert_messages_to_prompt +from litellm.types.llms.bedrock import * + + +class AmazonCohereChatConfig: + """ + Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html + """ + + documents: Optional[List[Document]] = None + search_queries_only: Optional[bool] = None + preamble: Optional[str] = None + max_tokens: Optional[int] = None + temperature: Optional[float] = None + p: Optional[float] = None + k: Optional[float] = None + prompt_truncation: Optional[str] = None + frequency_penalty: Optional[float] = None + presence_penalty: Optional[float] = None + seed: Optional[int] = None + return_prompt: Optional[bool] = None + stop_sequences: Optional[List[str]] = None + raw_prompting: Optional[bool] = None + + def __init__( + self, + documents: Optional[List[Document]] = None, + search_queries_only: Optional[bool] = None, + preamble: Optional[str] = None, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + p: Optional[float] = None, + k: Optional[float] = None, + prompt_truncation: Optional[str] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + seed: Optional[int] = None, + return_prompt: Optional[bool] = None, + stop_sequences: Optional[str] = None, + raw_prompting: Optional[bool] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self) -> List[str]: + return [ + "max_tokens", + "stream", + "stop", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "seed", + "stop", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["max_tokens"] = value + if param == "stream": + optional_params["stream"] = value + if param == "stop": + if isinstance(value, str): + value = [value] + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["p"] = value + if param == "frequency_penalty": + optional_params["frequency_penalty"] = value + if param == "presence_penalty": + optional_params["presence_penalty"] = value + if "seed": + optional_params["seed"] = value + return optional_params + + +class BedrockLLM(BaseLLM): + """ + Example call + + ``` + curl --location --request POST 'https://bedrock-runtime.{aws_region_name}.amazonaws.com/model/{bedrock_model_name}/invoke' \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + --user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \ + --aws-sigv4 "aws:amz:us-east-1:bedrock" \ + --data-raw '{ + "prompt": "Hi", + "temperature": 0, + "p": 0.9, + "max_tokens": 4096 + }' + ``` + """ + + def __init__(self) -> None: + super().__init__() + + def convert_messages_to_prompt( + self, model, messages, provider, custom_prompt_dict + ) -> Tuple[str, Optional[list]]: + # handle anthropic prompts and amazon titan prompts + prompt = "" + chat_history: Optional[list] = None + if provider == "anthropic" or provider == "amazon": + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details["initial_prompt_value"], + final_prompt_value=model_prompt_details["final_prompt_value"], + messages=messages, + ) + else: + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "mistral": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "meta": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "cohere": + prompt, chat_history = cohere_message_pt(messages=messages) + else: + prompt = "" + for message in messages: + if "role" in message: + if message["role"] == "user": + prompt += f"{message['content']}" + else: + prompt += f"{message['content']}" + else: + prompt += f"{message['content']}" + return prompt, chat_history # type: ignore + + def get_credentials( + self, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_region_name: Optional[str] = None, + aws_session_name: Optional[str] = None, + aws_profile_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + ): + """ + Return a boto3.Credentials object + """ + import boto3 + + ## CHECK IS 'os.environ/' passed in + params_to_check: List[Optional[str]] = [ + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + ] + + # Iterate over parameters and update if needed + for i, param in enumerate(params_to_check): + if param and param.startswith("os.environ/"): + _v = get_secret(param) + if _v is not None and isinstance(_v, str): + params_to_check[i] = _v + # Assign updated values back to parameters + ( + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + ) = params_to_check + + ### CHECK STS ### + if aws_role_name is not None and aws_session_name is not None: + sts_client = boto3.client( + "sts", + aws_access_key_id=aws_access_key_id, # [OPTIONAL] + aws_secret_access_key=aws_secret_access_key, # [OPTIONAL] + ) + + sts_response = sts_client.assume_role( + RoleArn=aws_role_name, RoleSessionName=aws_session_name + ) + + return sts_response["Credentials"] + elif aws_profile_name is not None: ### CHECK SESSION ### + # uses auth values from AWS profile usually stored in ~/.aws/credentials + client = boto3.Session(profile_name=aws_profile_name) + + return client.get_credentials() + else: + session = boto3.Session( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region_name, + ) + + return session.get_credentials() + + def process_response( + self, + model: str, + response: Union[requests.Response, httpx.Response], + model_response: ModelResponse, + stream: bool, + logging_obj: Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: List, + print_verbose, + encoding, + ) -> ModelResponse: + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + + ## RESPONSE OBJECT + try: + completion_response = response.json() + except: + raise BedrockError(message=response.text, status_code=422) + + try: + model_response.choices[0].message.content = completion_response["text"] # type: ignore + except Exception as e: + raise BedrockError(message=response.text, status_code=422) + + ## CALCULATING USAGE - bedrock returns usage in the headers + prompt_tokens = int( + response.headers.get( + "x-amzn-bedrock-input-token-count", + len(encoding.encode("".join(m.get("content", "") for m in messages))), + ) + ) + completion_tokens = int( + response.headers.get( + "x-amzn-bedrock-output-token-count", + len( + encoding.encode( + model_response.choices[0].message.content, # type: ignore + disallowed_special=(), + ) + ), + ) + ) + + model_response["created"] = int(time.time()) + model_response["model"] = model + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + setattr(model_response, "usage", usage) + + return model_response + + def completion( + self, + model: str, + messages: list, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj, + optional_params: dict, + acompletion: bool, + timeout: Optional[Union[float, httpx.Timeout]], + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + try: + import boto3 + + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + except ImportError as e: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + ## SETUP ## + stream = optional_params.pop("stream", None) + + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_region_name = optional_params.pop("aws_region_name", None) + aws_role_name = optional_params.pop("aws_role_name", None) + aws_session_name = optional_params.pop("aws_session_name", None) + aws_profile_name = optional_params.pop("aws_profile_name", None) + aws_bedrock_runtime_endpoint = optional_params.pop( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + + ### SET REGION NAME ### + if aws_region_name is None: + # check env # + litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) + + if litellm_aws_region_name is not None and isinstance( + litellm_aws_region_name, str + ): + aws_region_name = litellm_aws_region_name + + standard_aws_region_name = get_secret("AWS_REGION", None) + if standard_aws_region_name is not None and isinstance( + standard_aws_region_name, str + ): + aws_region_name = standard_aws_region_name + + if aws_region_name is None: + aws_region_name = "us-west-2" + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + ) + + ### SET RUNTIME ENDPOINT ### + endpoint_url = "" + env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT") + if aws_bedrock_runtime_endpoint is not None and isinstance( + aws_bedrock_runtime_endpoint, str + ): + endpoint_url = aws_bedrock_runtime_endpoint + elif env_aws_bedrock_runtime_endpoint and isinstance( + env_aws_bedrock_runtime_endpoint, str + ): + endpoint_url = env_aws_bedrock_runtime_endpoint + else: + endpoint_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" + + if stream is not None and stream == True: + endpoint_url = f"{endpoint_url}/model/{model}/invoke-with-response-stream" + else: + endpoint_url = f"{endpoint_url}/model/{model}/invoke" + + sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) + + provider = model.split(".")[0] + prompt, chat_history = self.convert_messages_to_prompt( + model, messages, provider, custom_prompt_dict + ) + inference_params = copy.deepcopy(optional_params) + + if provider == "cohere": + if model.startswith("cohere.command-r"): + ## LOAD CONFIG + config = litellm.AmazonCohereChatConfig().get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + _data = {"message": prompt, **inference_params} + if chat_history is not None: + _data["chat_history"] = chat_history + data = json.dumps(_data) + else: + ## LOAD CONFIG + config = litellm.AmazonCohereConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + if stream == True: + inference_params["stream"] = ( + True # cohere requires stream = True in inference params + ) + data = json.dumps({"prompt": prompt, **inference_params}) + else: + raise Exception("UNSUPPORTED PROVIDER") + + ## COMPLETION CALL + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + request = AWSRequest( + method="POST", url=endpoint_url, data=data, headers=headers + ) + sigv4.add_auth(request) + prepped = request.prepare() + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": prepped.url, + "headers": prepped.headers, + }, + ) + + ### ROUTING (ASYNC, STREAMING, SYNC) + if acompletion: + if isinstance(client, HTTPHandler): + client = None + if stream: + return self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=prepped.url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) # type: ignore + ### ASYNC COMPLETION + return self.async_completion( + model=model, + messages=messages, + data=data, + api_base=prepped.url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=False, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) # type: ignore + + if client is None or isinstance(client, AsyncHTTPHandler): + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + self.client = HTTPHandler(**_params) # type: ignore + else: + self.client = client + if stream is not None and stream == True: + response = self.client.post( + url=prepped.url, + headers=prepped.headers, # type: ignore + data=data, + stream=stream, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=response.text + ) + + decoder = AWSEventStreamDecoder() + + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + return streaming_response + + response = self.client.post(url=prepped.url, headers=prepped.headers, data=data) # type: ignore + + try: + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=response.text) + + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + optional_params=optional_params, + api_key="", + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + ) + + async def async_completion( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + data: str, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj, + stream, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + ) -> ModelResponse: + if client is None: + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + self.client = AsyncHTTPHandler(**_params) # type: ignore + else: + self.client = client # type: ignore + + response = await self.client.post(api_base, headers=headers, data=data) # type: ignore + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + api_key="", + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, + ) + + async def async_streaming( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + data: str, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj, + stream, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + ) -> CustomStreamWrapper: + if client is None: + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + self.client = AsyncHTTPHandler(**_params) # type: ignore + else: + self.client = client # type: ignore + + response = await self.client.post(api_base, headers=headers, data=data, stream=True) # type: ignore + + if response.status_code != 200: + raise BedrockError(status_code=response.status_code, message=response.text) + + decoder = AWSEventStreamDecoder() + + completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024)) + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + return streaming_response + + def embedding(self, *args, **kwargs): + return super().embedding(*args, **kwargs) + + +def get_response_stream_shape(): + from botocore.model import ServiceModel + from botocore.loaders import Loader + + loader = Loader() + bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2") + bedrock_service_model = ServiceModel(bedrock_service_dict) + return bedrock_service_model.shape_for("ResponseStream") + + +class AWSEventStreamDecoder: + def __init__(self) -> None: + from botocore.parsers import EventStreamJSONParser + + self.parser = EventStreamJSONParser() + + def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[GenericStreamingChunk]: + """Given an iterator that yields lines, iterate over it & yield every event encountered""" + from botocore.eventstream import EventStreamBuffer + + event_stream_buffer = EventStreamBuffer() + for chunk in iterator: + event_stream_buffer.add_data(chunk) + for event in event_stream_buffer: + message = self._parse_message_from_event(event) + if message: + # sse_event = ServerSentEvent(data=message, event="completion") + _data = json.loads(message) + streaming_chunk: GenericStreamingChunk = GenericStreamingChunk( + text=_data.get("text", ""), + is_finished=_data.get("is_finished", False), + finish_reason=_data.get("finish_reason", ""), + ) + yield streaming_chunk + + async def aiter_bytes( + self, iterator: AsyncIterator[bytes] + ) -> AsyncIterator[GenericStreamingChunk]: + """Given an async iterator that yields lines, iterate over it & yield every event encountered""" + from botocore.eventstream import EventStreamBuffer + + event_stream_buffer = EventStreamBuffer() + async for chunk in iterator: + event_stream_buffer.add_data(chunk) + for event in event_stream_buffer: + message = self._parse_message_from_event(event) + if message: + _data = json.loads(message) + streaming_chunk: GenericStreamingChunk = GenericStreamingChunk( + text=_data.get("text", ""), + is_finished=_data.get("is_finished", False), + finish_reason=_data.get("finish_reason", ""), + ) + yield streaming_chunk + + def _parse_message_from_event(self, event) -> Optional[str]: + response_dict = event.to_response_dict() + parsed_response = self.parser.parse(response_dict, get_response_stream_shape()) + if response_dict["status_code"] != 200: + raise ValueError(f"Bad response code, expected 200: {response_dict}") + + chunk = parsed_response.get("chunk") + if not chunk: + return None + + return chunk.get("bytes").decode() # type: ignore[no-any-return] diff --git a/litellm/llms/clarifai.py b/litellm/llms/clarifai.py new file mode 100644 index 000000000..e07a8d9e8 --- /dev/null +++ b/litellm/llms/clarifai.py @@ -0,0 +1,328 @@ +import os, types, traceback +import json +import requests +import time +from typing import Callable, Optional +from litellm.utils import ModelResponse, Usage, Choices, Message, CustomStreamWrapper +import litellm +import httpx +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from .prompt_templates.factory import prompt_factory, custom_prompt + + +class ClarifaiError(Exception): + def __init__(self, status_code, message, url): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", url=url + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) + +class ClarifaiConfig: + """ + Reference: https://clarifai.com/meta/Llama-2/models/llama2-70b-chat + TODO fill in the details + """ + max_tokens: Optional[int] = None + temperature: Optional[int] = None + top_k: Optional[int] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + temperature: Optional[int] = None, + top_k: Optional[int] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + +def validate_environment(api_key): + headers = { + "accept": "application/json", + "content-type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + +def completions_to_model(payload): + # if payload["n"] != 1: + # raise HTTPException( + # status_code=422, + # detail="Only one generation is supported. Please set candidate_count to 1.", + # ) + + params = {} + if temperature := payload.get("temperature"): + params["temperature"] = temperature + if max_tokens := payload.get("max_tokens"): + params["max_tokens"] = max_tokens + return { + "inputs": [{"data": {"text": {"raw": payload["prompt"]}}}], + "model": {"output_info": {"params": params}}, +} + +def process_response( + model, + prompt, + response, + model_response, + api_key, + data, + encoding, + logging_obj + ): + logging_obj.post_call( + input=prompt, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + ## RESPONSE OBJECT + try: + completion_response = response.json() + except Exception: + raise ClarifaiError( + message=response.text, status_code=response.status_code, url=model + ) + # print(completion_response) + try: + choices_list = [] + for idx, item in enumerate(completion_response["outputs"]): + if len(item["data"]["text"]["raw"]) > 0: + message_obj = Message(content=item["data"]["text"]["raw"]) + else: + message_obj = Message(content=None) + choice_obj = Choices( + finish_reason="stop", + index=idx + 1, #check + message=message_obj, + ) + choices_list.append(choice_obj) + model_response["choices"] = choices_list + + except Exception as e: + raise ClarifaiError( + message=traceback.format_exc(), status_code=response.status_code, url=model + ) + + # Calculate Usage + prompt_tokens = len(encoding.encode(prompt)) + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content")) + ) + model_response["model"] = model + model_response["usage"] = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + return model_response + +def convert_model_to_url(model: str, api_base: str): + user_id, app_id, model_id = model.split(".") + return f"{api_base}/users/{user_id}/apps/{app_id}/models/{model_id}/outputs" + +def get_prompt_model_name(url: str): + clarifai_model_name = url.split("/")[-2] + if "claude" in clarifai_model_name: + return "anthropic", clarifai_model_name.replace("_", ".") + if ("llama" in clarifai_model_name)or ("mistral" in clarifai_model_name): + return "", "meta-llama/llama-2-chat" + else: + return "", clarifai_model_name + +async def async_completion( + model: str, + prompt: str, + api_base: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + data=None, + optional_params=None, + litellm_params=None, + logger_fn=None, + headers={}): + + async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + response = await async_handler.post( + api_base, headers=headers, data=json.dumps(data) + ) + + return process_response( + model=model, + prompt=prompt, + response=response, + model_response=model_response, + api_key=api_key, + data=data, + encoding=encoding, + logging_obj=logging_obj, + ) + +def completion( + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + custom_prompt_dict={}, + acompletion=False, + optional_params=None, + litellm_params=None, + logger_fn=None, +): + headers = validate_environment(api_key) + model = convert_model_to_url(model, api_base) + prompt = " ".join(message["content"] for message in messages) # TODO + + ## Load Config + config = litellm.ClarifaiConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): + optional_params[k] = v + + custom_llm_provider, orig_model_name = get_prompt_model_name(model) + if custom_llm_provider == "anthropic": + prompt = prompt_factory( + model=orig_model_name, + messages=messages, + api_key=api_key, + custom_llm_provider="clarifai" + ) + else: + prompt = prompt_factory( + model=orig_model_name, + messages=messages, + api_key=api_key, + custom_llm_provider=custom_llm_provider + ) + # print(prompt); exit(0) + + data = { + "prompt": prompt, + **optional_params, + } + data = completions_to_model(data) + + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": api_base, + }, + ) + if acompletion==True: + return async_completion( + model=model, + prompt=prompt, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + data=data, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) + else: + ## COMPLETION CALL + response = requests.post( + model, + headers=headers, + data=json.dumps(data), + ) + # print(response.content); exit() + + if response.status_code != 200: + raise ClarifaiError(status_code=response.status_code, message=response.text, url=model) + + if "stream" in optional_params and optional_params["stream"] == True: + completion_stream = response.iter_lines() + stream_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="clarifai", + logging_obj=logging_obj, + ) + return stream_response + + else: + return process_response( + model=model, + prompt=prompt, + response=response, + model_response=model_response, + api_key=api_key, + data=data, + encoding=encoding, + logging_obj=logging_obj) + + +class ModelResponseIterator: + def __init__(self, model_response): + self.model_response = model_response + self.is_done = False + + # Sync iterator + def __iter__(self): + return self + + def __next__(self): + if self.is_done: + raise StopIteration + self.is_done = True + return self.model_response + + # Async iterator + def __aiter__(self): + return self + + async def __anext__(self): + if self.is_done: + raise StopAsyncIteration + self.is_done = True + return self.model_response \ No newline at end of file diff --git a/litellm/llms/cloudflare.py b/litellm/llms/cloudflare.py index b8187cbc9..5a24b3b44 100644 --- a/litellm/llms/cloudflare.py +++ b/litellm/llms/cloudflare.py @@ -1,11 +1,11 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional import litellm -import httpx +import httpx # type: ignore from litellm.utils import ModelResponse, Usage from .prompt_templates.factory import prompt_factory, custom_prompt diff --git a/litellm/llms/cohere.py b/litellm/llms/cohere.py index b867559c3..0ebdf38f1 100644 --- a/litellm/llms/cohere.py +++ b/litellm/llms/cohere.py @@ -1,12 +1,12 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time, traceback from typing import Callable, Optional from litellm.utils import ModelResponse, Choices, Message, Usage import litellm -import httpx +import httpx # type: ignore class CohereError(Exception): diff --git a/litellm/llms/cohere_chat.py b/litellm/llms/cohere_chat.py index 2a9bc320b..e4de6ddcb 100644 --- a/litellm/llms/cohere_chat.py +++ b/litellm/llms/cohere_chat.py @@ -1,12 +1,12 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time, traceback from typing import Callable, Optional from litellm.utils import ModelResponse, Choices, Message, Usage import litellm -import httpx +import httpx # type: ignore from .prompt_templates.factory import cohere_message_pt diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 7c7d4938a..0adbd95bf 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -58,16 +58,25 @@ class AsyncHTTPHandler: class HTTPHandler: def __init__( - self, timeout: httpx.Timeout = _DEFAULT_TIMEOUT, concurrent_limit=1000 + self, + timeout: Optional[httpx.Timeout] = None, + concurrent_limit=1000, + client: Optional[httpx.Client] = None, ): - # Create a client with a connection pool - self.client = httpx.Client( - timeout=timeout, - limits=httpx.Limits( - max_connections=concurrent_limit, - max_keepalive_connections=concurrent_limit, - ), - ) + if timeout is None: + timeout = _DEFAULT_TIMEOUT + + if client is None: + # Create a client with a connection pool + self.client = httpx.Client( + timeout=timeout, + limits=httpx.Limits( + max_connections=concurrent_limit, + max_keepalive_connections=concurrent_limit, + ), + ) + else: + self.client = client def close(self): # Close the client when you're done with it @@ -82,11 +91,15 @@ class HTTPHandler: def post( self, url: str, - data: Optional[dict] = None, + data: Optional[Union[dict, str]] = None, params: Optional[dict] = None, headers: Optional[dict] = None, + stream: bool = False, ): - response = self.client.post(url, data=data, params=params, headers=headers) + req = self.client.build_request( + "POST", url, data=data, params=params, headers=headers # type: ignore + ) + response = self.client.send(req, stream=stream) return response def __del__(self) -> None: diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 293773289..ad3c570e7 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -6,10 +6,12 @@ import httpx, requests from .base import BaseLLM import time import litellm -from typing import Callable, Dict, List, Any +from typing import Callable, Dict, List, Any, Literal from litellm.utils import ModelResponse, Choices, Message, CustomStreamWrapper, Usage from typing import Optional from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.types.completion import ChatCompletionMessageToolCallParam +import enum class HuggingfaceError(Exception): @@ -39,11 +41,29 @@ class HuggingfaceError(Exception): ) # Call the base class constructor with the parameters it needs +hf_task_list = [ + "text-generation-inference", + "conversational", + "text-classification", + "text-generation", +] + +hf_tasks = Literal[ + "text-generation-inference", + "conversational", + "text-classification", + "text-generation", +] + + class HuggingfaceConfig: """ Reference: https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/compat_generate """ + hf_task: Optional[hf_tasks] = ( + None # litellm-specific param, used to know the api spec to use when calling huggingface api + ) best_of: Optional[int] = None decoder_input_details: Optional[bool] = None details: Optional[bool] = True # enables returning logprobs + best of @@ -101,6 +121,51 @@ class HuggingfaceConfig: and v is not None } + def get_supported_openai_params(self): + return [ + "stream", + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "echo", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for param, value in non_default_params.items(): + # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None + if param == "temperature": + if value == 0.0 or value == 0: + # hugging face exception raised when temp==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive + value = 0.01 + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "n": + optional_params["best_of"] = value + optional_params["do_sample"] = ( + True # Need to sample if you want best of for hf inference endpoints + ) + if param == "stream": + optional_params["stream"] = value + if param == "stop": + optional_params["stop"] = value + if param == "max_tokens": + # HF TGI raises the following exception when max_new_tokens==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive + if value == 0: + value = 1 + optional_params["max_new_tokens"] = value + if param == "echo": + # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details + # Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False + optional_params["decoder_input_details"] = True + return optional_params + def output_parser(generated_text: str): """ @@ -162,16 +227,18 @@ def read_tgi_conv_models(): return set(), set() -def get_hf_task_for_model(model): +def get_hf_task_for_model(model: str) -> hf_tasks: # read text file, cast it to set # read the file called "huggingface_llms_metadata/hf_text_generation_models.txt" + if model.split("/")[0] in hf_task_list: + return model.split("/")[0] # type: ignore tgi_models, conversational_models = read_tgi_conv_models() if model in tgi_models: return "text-generation-inference" elif model in conversational_models: return "conversational" elif "roneneldan/TinyStories" in model: - return None + return "text-generation" else: return "text-generation-inference" # default to tgi @@ -202,7 +269,7 @@ class Huggingface(BaseLLM): self, completion_response, model_response, - task, + task: hf_tasks, optional_params, encoding, input_text, @@ -270,6 +337,10 @@ class Huggingface(BaseLLM): ) choices_list.append(choice_obj) model_response["choices"].extend(choices_list) + elif task == "text-classification": + model_response["choices"][0]["message"]["content"] = json.dumps( + completion_response + ) else: if len(completion_response[0]["generated_text"]) > 0: model_response["choices"][0]["message"]["content"] = output_parser( @@ -322,9 +393,9 @@ class Huggingface(BaseLLM): encoding, api_key, logging_obj, + optional_params: dict, custom_prompt_dict={}, acompletion: bool = False, - optional_params=None, litellm_params=None, logger_fn=None, ): @@ -333,6 +404,12 @@ class Huggingface(BaseLLM): try: headers = self.validate_environment(api_key, headers) task = get_hf_task_for_model(model) + ## VALIDATE API FORMAT + if task is None or not isinstance(task, str) or task not in hf_task_list: + raise Exception( + "Invalid hf task - {}. Valid formats - {}.".format(task, hf_tasks) + ) + print_verbose(f"{model}, {task}") completion_url = "" input_text = "" @@ -399,10 +476,11 @@ class Huggingface(BaseLLM): data = { "inputs": prompt, "parameters": optional_params, - "stream": ( + "stream": ( # type: ignore True if "stream" in optional_params - and optional_params["stream"] == True + and isinstance(optional_params["stream"], bool) + and optional_params["stream"] == True # type: ignore else False ), } @@ -432,14 +510,15 @@ class Huggingface(BaseLLM): inference_params.pop("return_full_text") data = { "inputs": prompt, - "parameters": inference_params, - "stream": ( + } + if task == "text-generation-inference": + data["parameters"] = inference_params + data["stream"] = ( # type: ignore True if "stream" in optional_params and optional_params["stream"] == True else False - ), - } + ) input_text = prompt ## LOGGING logging_obj.pre_call( @@ -530,10 +609,10 @@ class Huggingface(BaseLLM): isinstance(completion_response, dict) and "error" in completion_response ): - print_verbose(f"completion error: {completion_response['error']}") + print_verbose(f"completion error: {completion_response['error']}") # type: ignore print_verbose(f"response.status_code: {response.status_code}") raise HuggingfaceError( - message=completion_response["error"], + message=completion_response["error"], # type: ignore status_code=response.status_code, ) return self.convert_to_model_response_object( @@ -562,7 +641,7 @@ class Huggingface(BaseLLM): data: dict, headers: dict, model_response: ModelResponse, - task: str, + task: hf_tasks, encoding: Any, input_text: str, model: str, diff --git a/litellm/llms/maritalk.py b/litellm/llms/maritalk.py index 4c6b86d3c..dfe53e9df 100644 --- a/litellm/llms/maritalk.py +++ b/litellm/llms/maritalk.py @@ -1,7 +1,7 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time, traceback from typing import Callable, Optional, List from litellm.utils import ModelResponse, Choices, Message, Usage diff --git a/litellm/llms/nlp_cloud.py b/litellm/llms/nlp_cloud.py index 86648118f..cd5f17a90 100644 --- a/litellm/llms/nlp_cloud.py +++ b/litellm/llms/nlp_cloud.py @@ -1,7 +1,7 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional import litellm diff --git a/litellm/llms/ollama.py b/litellm/llms/ollama.py index 5972d9e8c..9c9b5e898 100644 --- a/litellm/llms/ollama.py +++ b/litellm/llms/ollama.py @@ -1,9 +1,10 @@ -import requests, types, time +from itertools import chain +import requests, types, time # type: ignore import json, uuid import traceback from typing import Optional import litellm -import httpx, aiohttp, asyncio +import httpx, aiohttp, asyncio # type: ignore from .prompt_templates.factory import prompt_factory, custom_prompt @@ -212,25 +213,31 @@ def get_ollama_response( ## RESPONSE OBJECT model_response["choices"][0]["finish_reason"] = "stop" - if optional_params.get("format", "") == "json": + if data.get("format", "") == "json": function_call = json.loads(response_json["response"]) message = litellm.Message( content=None, tool_calls=[ { "id": f"call_{str(uuid.uuid4())}", - "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, "type": "function", } ], ) model_response["choices"][0]["message"] = message + model_response["choices"][0]["finish_reason"] = "tool_calls" else: model_response["choices"][0]["message"]["content"] = response_json["response"] model_response["created"] = int(time.time()) model_response["model"] = "ollama/" + model prompt_tokens = response_json.get("prompt_eval_count", len(encoding.encode(prompt, disallowed_special=()))) # type: ignore - completion_tokens = response_json.get("eval_count", len(response_json.get("message",dict()).get("content", ""))) + completion_tokens = response_json.get( + "eval_count", len(response_json.get("message", dict()).get("content", "")) + ) model_response["usage"] = litellm.Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, @@ -255,8 +262,37 @@ def ollama_completion_stream(url, data, logging_obj): custom_llm_provider="ollama", logging_obj=logging_obj, ) - for transformed_chunk in streamwrapper: - yield transformed_chunk + # If format is JSON, this was a function call + # Gather all chunks and return the function call as one delta to simplify parsing + if data.get("format", "") == "json": + first_chunk = next(streamwrapper) + response_content = "".join( + chunk.choices[0].delta.content + for chunk in chain([first_chunk], streamwrapper) + if chunk.choices[0].delta.content + ) + + function_call = json.loads(response_content) + delta = litellm.utils.Delta( + content=None, + tool_calls=[ + { + "id": f"call_{str(uuid.uuid4())}", + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, + "type": "function", + } + ], + ) + model_response = first_chunk + model_response["choices"][0]["delta"] = delta + model_response["choices"][0]["finish_reason"] = "tool_calls" + yield model_response + else: + for transformed_chunk in streamwrapper: + yield transformed_chunk except Exception as e: raise e @@ -278,8 +314,40 @@ async def ollama_async_streaming(url, data, model_response, encoding, logging_ob custom_llm_provider="ollama", logging_obj=logging_obj, ) - async for transformed_chunk in streamwrapper: - yield transformed_chunk + + # If format is JSON, this was a function call + # Gather all chunks and return the function call as one delta to simplify parsing + if data.get("format", "") == "json": + first_chunk = await anext(streamwrapper) + first_chunk_content = first_chunk.choices[0].delta.content or "" + response_content = first_chunk_content + "".join( + [ + chunk.choices[0].delta.content + async for chunk in streamwrapper + if chunk.choices[0].delta.content + ] + ) + function_call = json.loads(response_content) + delta = litellm.utils.Delta( + content=None, + tool_calls=[ + { + "id": f"call_{str(uuid.uuid4())}", + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, + "type": "function", + } + ], + ) + model_response = first_chunk + model_response["choices"][0]["delta"] = delta + model_response["choices"][0]["finish_reason"] = "tool_calls" + yield model_response + else: + async for transformed_chunk in streamwrapper: + yield transformed_chunk except Exception as e: traceback.print_exc() raise e @@ -317,12 +385,16 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj): tool_calls=[ { "id": f"call_{str(uuid.uuid4())}", - "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, "type": "function", } ], ) model_response["choices"][0]["message"] = message + model_response["choices"][0]["finish_reason"] = "tool_calls" else: model_response["choices"][0]["message"]["content"] = response_json[ "response" @@ -330,7 +402,10 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj): model_response["created"] = int(time.time()) model_response["model"] = "ollama/" + data["model"] prompt_tokens = response_json.get("prompt_eval_count", len(encoding.encode(data["prompt"], disallowed_special=()))) # type: ignore - completion_tokens = response_json.get("eval_count", len(response_json.get("message",dict()).get("content", ""))) + completion_tokens = response_json.get( + "eval_count", + len(response_json.get("message", dict()).get("content", "")), + ) model_response["usage"] = litellm.Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, @@ -417,3 +492,25 @@ async def ollama_aembeddings( "total_tokens": total_input_tokens, } return model_response + + +def ollama_embeddings( + api_base: str, + model: str, + prompts: list, + optional_params=None, + logging_obj=None, + model_response=None, + encoding=None, +): + return asyncio.run( + ollama_aembeddings( + api_base, + model, + prompts, + optional_params, + logging_obj, + model_response, + encoding, + ) + ) diff --git a/litellm/llms/ollama_chat.py b/litellm/llms/ollama_chat.py index fb0d76b05..d1ff4953f 100644 --- a/litellm/llms/ollama_chat.py +++ b/litellm/llms/ollama_chat.py @@ -1,3 +1,4 @@ +from itertools import chain import requests, types, time import json, uuid import traceback @@ -297,8 +298,9 @@ def get_ollama_response( ], ) model_response["choices"][0]["message"] = message + model_response["choices"][0]["finish_reason"] = "tool_calls" else: - model_response["choices"][0]["message"] = response_json["message"] + model_response["choices"][0]["message"]["content"] = response_json["message"]["content"] model_response["created"] = int(time.time()) model_response["model"] = "ollama/" + model prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore @@ -335,8 +337,35 @@ def ollama_completion_stream(url, api_key, data, logging_obj): custom_llm_provider="ollama_chat", logging_obj=logging_obj, ) - for transformed_chunk in streamwrapper: - yield transformed_chunk + + # If format is JSON, this was a function call + # Gather all chunks and return the function call as one delta to simplify parsing + if data.get("format", "") == "json": + first_chunk = next(streamwrapper) + response_content = "".join( + chunk.choices[0].delta.content + for chunk in chain([first_chunk], streamwrapper) + if chunk.choices[0].delta.content + ) + + function_call = json.loads(response_content) + delta = litellm.utils.Delta( + content=None, + tool_calls=[ + { + "id": f"call_{str(uuid.uuid4())}", + "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "type": "function", + } + ], + ) + model_response = first_chunk + model_response["choices"][0]["delta"] = delta + model_response["choices"][0]["finish_reason"] = "tool_calls" + yield model_response + else: + for transformed_chunk in streamwrapper: + yield transformed_chunk except Exception as e: raise e @@ -366,8 +395,36 @@ async def ollama_async_streaming( custom_llm_provider="ollama_chat", logging_obj=logging_obj, ) - async for transformed_chunk in streamwrapper: - yield transformed_chunk + + # If format is JSON, this was a function call + # Gather all chunks and return the function call as one delta to simplify parsing + if data.get("format", "") == "json": + first_chunk = await anext(streamwrapper) + first_chunk_content = first_chunk.choices[0].delta.content or "" + response_content = first_chunk_content + "".join( + [ + chunk.choices[0].delta.content + async for chunk in streamwrapper + if chunk.choices[0].delta.content] + ) + function_call = json.loads(response_content) + delta = litellm.utils.Delta( + content=None, + tool_calls=[ + { + "id": f"call_{str(uuid.uuid4())}", + "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "type": "function", + } + ], + ) + model_response = first_chunk + model_response["choices"][0]["delta"] = delta + model_response["choices"][0]["finish_reason"] = "tool_calls" + yield model_response + else: + async for transformed_chunk in streamwrapper: + yield transformed_chunk except Exception as e: traceback.print_exc() @@ -425,8 +482,9 @@ async def ollama_acompletion( ], ) model_response["choices"][0]["message"] = message + model_response["choices"][0]["finish_reason"] = "tool_calls" else: - model_response["choices"][0]["message"] = response_json["message"] + model_response["choices"][0]["message"]["content"] = response_json["message"]["content"] model_response["created"] = int(time.time()) model_response["model"] = "ollama_chat/" + data["model"] diff --git a/litellm/llms/oobabooga.py b/litellm/llms/oobabooga.py index b166c9069..f8f32e0fe 100644 --- a/litellm/llms/oobabooga.py +++ b/litellm/llms/oobabooga.py @@ -1,7 +1,7 @@ import os import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional from litellm.utils import ModelResponse, Usage diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index f68ab235e..7acbdfae0 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -1,4 +1,13 @@ -from typing import Optional, Union, Any, BinaryIO +from typing import ( + Optional, + Union, + Any, + BinaryIO, + Literal, + Iterable, +) +from typing_extensions import override +from pydantic import BaseModel import types, time, json, traceback import httpx from .base import BaseLLM @@ -13,10 +22,10 @@ from litellm.utils import ( TextCompletionResponse, ) from typing import Callable, Optional -import aiohttp, requests import litellm from .prompt_templates.factory import prompt_factory, custom_prompt from openai import OpenAI, AsyncOpenAI +from ..types.llms.openai import * class OpenAIError(Exception): @@ -44,6 +53,113 @@ class OpenAIError(Exception): ) # Call the base class constructor with the parameters it needs +class MistralConfig: + """ + Reference: https://docs.mistral.ai/api/ + + The class `MistralConfig` provides configuration for the Mistral's Chat API interface. Below are the parameters: + + - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2. API Default - 0.7. + + - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. API Default - 1. + + - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion. API Default - null. + + - `tools` (list or null): A list of available tools for the model. Use this to specify functions for which the model can generate JSON inputs. + + - `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'. + + - `random_seed` (integer or null): The seed to use for random sampling. If set, different calls will generate deterministic results. + + - `safe_prompt` (boolean): Whether to inject a safety prompt before all conversations. API Default - 'false'. + + - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. + """ + + temperature: Optional[int] = None + top_p: Optional[int] = None + max_tokens: Optional[int] = None + tools: Optional[list] = None + tool_choice: Optional[Literal["auto", "any", "none"]] = None + random_seed: Optional[int] = None + safe_prompt: Optional[bool] = None + response_format: Optional[dict] = None + + def __init__( + self, + temperature: Optional[int] = None, + top_p: Optional[int] = None, + max_tokens: Optional[int] = None, + tools: Optional[list] = None, + tool_choice: Optional[Literal["auto", "any", "none"]] = None, + random_seed: Optional[int] = None, + safe_prompt: Optional[bool] = None, + response_format: Optional[dict] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return [ + "stream", + "temperature", + "top_p", + "max_tokens", + "tools", + "tool_choice", + "seed", + "response_format", + ] + + def _map_tool_choice(self, tool_choice: str) -> str: + if tool_choice == "auto" or tool_choice == "none": + return tool_choice + elif tool_choice == "required": + return "any" + else: # openai 'tool_choice' object param not supported by Mistral API + return "any" + + def map_openai_params(self, non_default_params: dict, optional_params: dict): + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["max_tokens"] = value + if param == "tools": + optional_params["tools"] = value + if param == "stream" and value == True: + optional_params["stream"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "tool_choice" and isinstance(value, str): + optional_params["tool_choice"] = self._map_tool_choice( + tool_choice=value + ) + if param == "seed": + optional_params["extra_body"] = {"random_seed": value} + return optional_params + + class OpenAIConfig: """ Reference: https://platform.openai.com/docs/api-reference/chat/create @@ -246,7 +362,7 @@ class OpenAIChatCompletion(BaseLLM): def completion( self, model_response: ModelResponse, - timeout: float, + timeout: Union[float, httpx.Timeout], model: Optional[str] = None, messages: Optional[list] = None, print_verbose: Optional[Callable] = None, @@ -271,9 +387,12 @@ class OpenAIChatCompletion(BaseLLM): if model is None or messages is None: raise OpenAIError(status_code=422, message=f"Missing model or messages") - if not isinstance(timeout, float): + if not isinstance(timeout, float) and not isinstance( + timeout, httpx.Timeout + ): raise OpenAIError( - status_code=422, message=f"Timeout needs to be a float" + status_code=422, + message=f"Timeout needs to be a float or httpx.Timeout", ) if custom_llm_provider != "openai": @@ -425,7 +544,7 @@ class OpenAIChatCompletion(BaseLLM): self, data: dict, model_response: ModelResponse, - timeout: float, + timeout: Union[float, httpx.Timeout], api_key: Optional[str] = None, api_base: Optional[str] = None, organization: Optional[str] = None, @@ -480,7 +599,7 @@ class OpenAIChatCompletion(BaseLLM): def streaming( self, logging_obj, - timeout: float, + timeout: Union[float, httpx.Timeout], data: dict, model: str, api_key: Optional[str] = None, @@ -518,13 +637,14 @@ class OpenAIChatCompletion(BaseLLM): model=model, custom_llm_provider="openai", logging_obj=logging_obj, + stream_options=data.get("stream_options", None), ) return streamwrapper async def async_streaming( self, logging_obj, - timeout: float, + timeout: Union[float, httpx.Timeout], data: dict, model: str, api_key: Optional[str] = None, @@ -567,6 +687,7 @@ class OpenAIChatCompletion(BaseLLM): model=model, custom_llm_provider="openai", logging_obj=logging_obj, + stream_options=data.get("stream_options", None), ) return streamwrapper except ( @@ -1191,6 +1312,7 @@ class OpenAITextCompletion(BaseLLM): model=model, custom_llm_provider="text-completion-openai", logging_obj=logging_obj, + stream_options=data.get("stream_options", None), ) for chunk in streamwrapper: @@ -1229,7 +1351,228 @@ class OpenAITextCompletion(BaseLLM): model=model, custom_llm_provider="text-completion-openai", logging_obj=logging_obj, + stream_options=data.get("stream_options", None), ) async for transformed_chunk in streamwrapper: yield transformed_chunk + + +class OpenAIAssistantsAPI(BaseLLM): + def __init__(self) -> None: + super().__init__() + + def get_openai_client( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI] = None, + ) -> OpenAI: + received_args = locals() + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client": + pass + elif k == "api_base" and v is not None: + data["base_url"] = v + elif v is not None: + data[k] = v + openai_client = OpenAI(**data) # type: ignore + else: + openai_client = client + + return openai_client + + ### ASSISTANTS ### + + def get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + ) -> SyncCursorPage[Assistant]: + openai_client = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = openai_client.beta.assistants.list() + + return response + + ### MESSAGES ### + + def add_message( + self, + thread_id: str, + message_data: MessageData, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI] = None, + ) -> OpenAIMessage: + + openai_client = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + thread_message: OpenAIMessage = openai_client.beta.threads.messages.create( # type: ignore + thread_id, **message_data # type: ignore + ) + + response_obj: Optional[OpenAIMessage] = None + if getattr(thread_message, "status", None) is None: + thread_message.status = "completed" + response_obj = OpenAIMessage(**thread_message.dict()) + else: + response_obj = OpenAIMessage(**thread_message.dict()) + return response_obj + + def get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI] = None, + ) -> SyncCursorPage[OpenAIMessage]: + openai_client = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = openai_client.beta.threads.messages.list(thread_id=thread_id) + + return response + + ### THREADS ### + + def create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + ) -> Thread: + """ + Here's an example: + ``` + from litellm.llms.openai import OpenAIAssistantsAPI, MessageData + + # create thread + message: MessageData = {"role": "user", "content": "Hey, how's it going?"} + openai_api.create_thread(messages=[message]) + ``` + """ + openai_client = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + data = {} + if messages is not None: + data["messages"] = messages # type: ignore + if metadata is not None: + data["metadata"] = metadata # type: ignore + + message_thread = openai_client.beta.threads.create(**data) # type: ignore + + return Thread(**message_thread.dict()) + + def get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + ) -> Thread: + openai_client = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = openai_client.beta.threads.retrieve(thread_id=thread_id) + + return Thread(**response.dict()) + + def delete_thread(self): + pass + + ### RUNS ### + + def run_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + ) -> Run: + openai_client = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = openai_client.beta.threads.runs.create_and_poll( # type: ignore + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + ) + + return response diff --git a/litellm/llms/petals.py b/litellm/llms/petals.py index 25403f598..334b80d38 100644 --- a/litellm/llms/petals.py +++ b/litellm/llms/petals.py @@ -1,7 +1,7 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional import litellm diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py new file mode 100644 index 000000000..1e7e1d334 --- /dev/null +++ b/litellm/llms/predibase.py @@ -0,0 +1,518 @@ +# What is this? +## Controller file for Predibase Integration - https://predibase.com/ + + +import os, types +import json +from enum import Enum +import requests, copy # type: ignore +import time +from typing import Callable, Optional, List, Literal, Union +from litellm.utils import ( + ModelResponse, + Usage, + map_finish_reason, + CustomStreamWrapper, + Message, + Choices, +) +import litellm +from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from .base import BaseLLM +import httpx # type: ignore + + +class PredibaseError(Exception): + def __init__( + self, + status_code, + message, + request: Optional[httpx.Request] = None, + response: Optional[httpx.Response] = None, + ): + self.status_code = status_code + self.message = message + if request is not None: + self.request = request + else: + self.request = httpx.Request( + method="POST", + url="https://docs.predibase.com/user-guide/inference/rest_api", + ) + if response is not None: + self.response = response + else: + self.response = httpx.Response( + status_code=status_code, request=self.request + ) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +class PredibaseConfig: + """ + Reference: https://docs.predibase.com/user-guide/inference/rest_api + + """ + + adapter_id: Optional[str] = None + adapter_source: Optional[Literal["pbase", "hub", "s3"]] = None + best_of: Optional[int] = None + decoder_input_details: Optional[bool] = None + details: bool = True # enables returning logprobs + best of + max_new_tokens: int = ( + 256 # openai default - requests hang if max_new_tokens not given + ) + repetition_penalty: Optional[float] = None + return_full_text: Optional[bool] = ( + False # by default don't return the input as part of the output + ) + seed: Optional[int] = None + stop: Optional[List[str]] = None + temperature: Optional[float] = None + top_k: Optional[int] = None + top_p: Optional[int] = None + truncate: Optional[int] = None + typical_p: Optional[float] = None + watermark: Optional[bool] = None + + def __init__( + self, + best_of: Optional[int] = None, + decoder_input_details: Optional[bool] = None, + details: Optional[bool] = None, + max_new_tokens: Optional[int] = None, + repetition_penalty: Optional[float] = None, + return_full_text: Optional[bool] = None, + seed: Optional[int] = None, + stop: Optional[List[str]] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[int] = None, + truncate: Optional[int] = None, + typical_p: Optional[float] = None, + watermark: Optional[bool] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"] + + +class PredibaseChatCompletion(BaseLLM): + def __init__(self) -> None: + super().__init__() + + def _validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: + if api_key is None: + raise ValueError( + "Missing Predibase API Key - A call is being made to predibase but no key is set either in the environment variables or via params" + ) + headers = { + "content-type": "application/json", + "Authorization": "Bearer {}".format(api_key), + } + if user_headers is not None and isinstance(user_headers, dict): + headers = {**headers, **user_headers} + return headers + + def output_parser(self, generated_text: str): + """ + Parse the output text to remove any special characters. In our current approach we just check for ChatML tokens. + + Initial issue that prompted this - https://github.com/BerriAI/litellm/issues/763 + """ + chat_template_tokens = [ + "<|assistant|>", + "<|system|>", + "<|user|>", + "", + "", + ] + for token in chat_template_tokens: + if generated_text.strip().startswith(token): + generated_text = generated_text.replace(token, "", 1) + if generated_text.endswith(token): + generated_text = generated_text[::-1].replace(token[::-1], "", 1)[::-1] + return generated_text + + def process_response( + self, + model: str, + response: Union[requests.Response, httpx.Response], + model_response: ModelResponse, + stream: bool, + logging_obj: litellm.utils.Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: list, + print_verbose, + encoding, + ) -> ModelResponse: + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + ## RESPONSE OBJECT + try: + completion_response = response.json() + except: + raise PredibaseError(message=response.text, status_code=422) + if "error" in completion_response: + raise PredibaseError( + message=str(completion_response["error"]), + status_code=response.status_code, + ) + else: + if ( + not isinstance(completion_response, dict) + or "generated_text" not in completion_response + ): + raise PredibaseError( + status_code=422, + message=f"response is not in expected format - {completion_response}", + ) + + if len(completion_response["generated_text"]) > 0: + model_response["choices"][0]["message"]["content"] = self.output_parser( + completion_response["generated_text"] + ) + ## GETTING LOGPROBS + FINISH REASON + if ( + "details" in completion_response + and "tokens" in completion_response["details"] + ): + model_response.choices[0].finish_reason = completion_response[ + "details" + ]["finish_reason"] + sum_logprob = 0 + for token in completion_response["details"]["tokens"]: + if token["logprob"] != None: + sum_logprob += token["logprob"] + model_response["choices"][0][ + "message" + ]._logprob = ( + sum_logprob # [TODO] move this to using the actual logprobs + ) + if "best_of" in optional_params and optional_params["best_of"] > 1: + if ( + "details" in completion_response + and "best_of_sequences" in completion_response["details"] + ): + choices_list = [] + for idx, item in enumerate( + completion_response["details"]["best_of_sequences"] + ): + sum_logprob = 0 + for token in item["tokens"]: + if token["logprob"] != None: + sum_logprob += token["logprob"] + if len(item["generated_text"]) > 0: + message_obj = Message( + content=self.output_parser(item["generated_text"]), + logprobs=sum_logprob, + ) + else: + message_obj = Message(content=None) + choice_obj = Choices( + finish_reason=item["finish_reason"], + index=idx + 1, + message=message_obj, + ) + choices_list.append(choice_obj) + model_response["choices"].extend(choices_list) + + ## CALCULATING USAGE + prompt_tokens = 0 + try: + prompt_tokens = len( + encoding.encode(model_response["choices"][0]["message"]["content"]) + ) ##[TODO] use a model-specific tokenizer here + except: + # this should remain non blocking we should not block a response returning if calculating usage fails + pass + output_text = model_response["choices"][0]["message"].get("content", "") + if output_text is not None and len(output_text) > 0: + completion_tokens = 0 + try: + completion_tokens = len( + encoding.encode( + model_response["choices"][0]["message"].get("content", "") + ) + ) ##[TODO] use a model-specific tokenizer + except: + # this should remain non blocking we should not block a response returning if calculating usage fails + pass + else: + completion_tokens = 0 + + total_tokens = prompt_tokens + completion_tokens + + model_response["created"] = int(time.time()) + model_response["model"] = model + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + model_response.usage = usage # type: ignore + return model_response + + def completion( + self, + model: str, + messages: list, + api_base: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key: str, + logging_obj, + optional_params: dict, + tenant_id: str, + acompletion=None, + litellm_params=None, + logger_fn=None, + headers: dict = {}, + ) -> Union[ModelResponse, CustomStreamWrapper]: + headers = self._validate_environment(api_key, headers) + completion_url = "" + input_text = "" + base_url = "https://serving.app.predibase.com" + if "https" in model: + completion_url = model + elif api_base: + base_url = api_base + elif "PREDIBASE_API_BASE" in os.environ: + base_url = os.getenv("PREDIBASE_API_BASE", "") + + completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}" + + if optional_params.get("stream", False) == True: + completion_url += "/generate_stream" + else: + completion_url += "/generate" + + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details["initial_prompt_value"], + final_prompt_value=model_prompt_details["final_prompt_value"], + messages=messages, + ) + else: + prompt = prompt_factory(model=model, messages=messages) + + ## Load Config + config = litellm.PredibaseConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + stream = optional_params.pop("stream", False) + + data = { + "inputs": prompt, + "parameters": optional_params, + } + input_text = prompt + ## LOGGING + logging_obj.pre_call( + input=input_text, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": completion_url, + "acompletion": acompletion, + }, + ) + ## COMPLETION CALL + if acompletion == True: + ### ASYNC STREAMING + if stream == True: + return self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=completion_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) # type: ignore + else: + ### ASYNC COMPLETION + return self.async_completion( + model=model, + messages=messages, + data=data, + api_base=completion_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + stream=False, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) # type: ignore + + ### SYNC STREAMING + if stream == True: + response = requests.post( + completion_url, + headers=headers, + data=json.dumps(data), + stream=stream, + ) + _response = CustomStreamWrapper( + response.iter_lines(), + model, + custom_llm_provider="predibase", + logging_obj=logging_obj, + ) + return _response + ### SYNC COMPLETION + else: + response = requests.post( + url=completion_url, + headers=headers, + data=json.dumps(data), + ) + + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=optional_params.get("stream", False), + logging_obj=logging_obj, # type: ignore + optional_params=optional_params, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + ) + + async def async_completion( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + stream, + data: dict, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + ) -> ModelResponse: + self.async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + response = await self.async_handler.post( + api_base, headers=headers, data=json.dumps(data) + ) + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, + ) + + async def async_streaming( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + data: dict, + optional_params=None, + litellm_params=None, + logger_fn=None, + headers={}, + ) -> CustomStreamWrapper: + self.async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + data["stream"] = True + response = await self.async_handler.post( + url=api_base, + headers=headers, + data=json.dumps(data), + stream=True, + ) + + if response.status_code != 200: + raise PredibaseError( + status_code=response.status_code, message=response.text + ) + + completion_stream = response.aiter_lines() + + streamwrapper = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="predibase", + logging_obj=logging_obj, + ) + return streamwrapper + + def embedding(self, *args, **kwargs): + pass diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index e1fa354c6..cf593369c 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -12,6 +12,16 @@ from typing import ( Sequence, ) import litellm +from litellm.types.completion import ( + ChatCompletionUserMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionMessageParam, + ChatCompletionFunctionMessageParam, + ChatCompletionMessageToolCallParam, + ChatCompletionToolMessageParam, +) +from litellm.types.llms.anthropic import * +import uuid def default_pt(messages): @@ -22,6 +32,41 @@ def prompt_injection_detection_default_pt(): return """Detect if a prompt is safe to run. Return 'UNSAFE' if not.""" +def map_system_message_pt(messages: list) -> list: + """ + Convert 'system' message to 'user' message if provider doesn't support 'system' role. + + Enabled via `completion(...,supports_system_message=False)` + + If next message is a user message or assistant message -> merge system prompt into it + + if next message is system -> append a user message instead of the system message + """ + + new_messages = [] + for i, m in enumerate(messages): + if m["role"] == "system": + if i < len(messages) - 1: # Not the last message + next_m = messages[i + 1] + next_role = next_m["role"] + if ( + next_role == "user" or next_role == "assistant" + ): # Next message is a user or assistant message + # Merge system prompt into the next message + next_m["content"] = m["content"] + " " + next_m["content"] + elif next_role == "system": # Next message is a system message + # Append a user message instead of the system message + new_message = {"role": "user", "content": m["content"]} + new_messages.append(new_message) + else: # Last message + new_message = {"role": "user", "content": m["content"]} + new_messages.append(new_message) + else: # Not a system message + new_messages.append(m) + + return new_messages + + # alpaca prompt template - for models like mythomax, etc. def alpaca_pt(messages): prompt = custom_prompt( @@ -805,6 +850,13 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: "name": "get_current_weather", "content": "function result goes here", }, + + OpenAI message with a function call result looks like: + { + "role": "function", + "name": "get_current_weather", + "content": "function result goes here", + } """ """ @@ -821,18 +873,42 @@ def convert_to_anthropic_tool_result(message: dict) -> dict: ] } """ - tool_call_id = message.get("tool_call_id") - content = message.get("content") + if message["role"] == "tool": + tool_call_id = message.get("tool_call_id") + content = message.get("content") - # We can't determine from openai message format whether it's a successful or - # error call result so default to the successful result template - anthropic_tool_result = { - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": content, - } + # We can't determine from openai message format whether it's a successful or + # error call result so default to the successful result template + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": tool_call_id, + "content": content, + } + return anthropic_tool_result + elif message["role"] == "function": + content = message.get("content") + anthropic_tool_result = { + "type": "tool_result", + "tool_use_id": str(uuid.uuid4()), + "content": content, + } + return anthropic_tool_result + return {} - return anthropic_tool_result + +def convert_function_to_anthropic_tool_invoke(function_call): + try: + anthropic_tool_invoke = [ + { + "type": "tool_use", + "id": str(uuid.uuid4()), + "name": get_attribute_or_key(function_call, "name"), + "input": json.loads(get_attribute_or_key(function_call, "arguments")), + } + ] + return anthropic_tool_invoke + except Exception as e: + raise e def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: @@ -895,7 +971,7 @@ def convert_to_anthropic_tool_invoke(tool_calls: list) -> list: def anthropic_messages_pt(messages: list): """ format messages for anthropic - 1. Anthropic supports roles like "user" and "assistant", (here litellm translates system-> assistant) + 1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately) 2. The first message always needs to be of role "user" 3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm) 4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise) @@ -903,12 +979,14 @@ def anthropic_messages_pt(messages: list): 6. Ensure we only accept role, content. (message.name is not supported) """ # add role=tool support to allow function call result/error submission - user_message_types = {"user", "tool"} + user_message_types = {"user", "tool", "function"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. - new_messages = [] + new_messages: list = [] msg_i = 0 + tool_use_param = False while msg_i < len(messages): user_content = [] + init_msg_i = msg_i ## MERGE CONSECUTIVE USER CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: if isinstance(messages[msg_i]["content"], list): @@ -924,7 +1002,10 @@ def anthropic_messages_pt(messages: list): ) elif m.get("type", "") == "text": user_content.append({"type": "text", "text": m["text"]}) - elif messages[msg_i]["role"] == "tool": + elif ( + messages[msg_i]["role"] == "tool" + or messages[msg_i]["role"] == "function" + ): # OpenAI's tool message content will always be a string user_content.append(convert_to_anthropic_tool_result(messages[msg_i])) else: @@ -953,11 +1034,24 @@ def anthropic_messages_pt(messages: list): convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"]) ) + if messages[msg_i].get("function_call"): + assistant_content.extend( + convert_function_to_anthropic_tool_invoke( + messages[msg_i]["function_call"] + ) + ) + msg_i += 1 if assistant_content: new_messages.append({"role": "assistant", "content": assistant_content}) + if msg_i == init_msg_i: # prevent infinite loops + raise Exception( + "Invalid Message passed in - {}. File an issue https://github.com/BerriAI/litellm/issues".format( + messages[msg_i] + ) + ) if not new_messages or new_messages[0]["role"] != "user": if litellm.modify_params: new_messages.insert( @@ -969,11 +1063,14 @@ def anthropic_messages_pt(messages: list): ) if new_messages[-1]["role"] == "assistant": - for content in new_messages[-1]["content"]: - if isinstance(content, dict) and content["type"] == "text": - content["text"] = content[ - "text" - ].rstrip() # no trailing whitespace for final assistant message + if isinstance(new_messages[-1]["content"], str): + new_messages[-1]["content"] = new_messages[-1]["content"].rstrip() + elif isinstance(new_messages[-1]["content"], list): + for content in new_messages[-1]["content"]: + if isinstance(content, dict) and content["type"] == "text": + content["text"] = content[ + "text" + ].rstrip() # no trailing whitespace for final assistant message return new_messages @@ -1412,6 +1509,11 @@ def prompt_factory( model="meta-llama/Meta-Llama-3-8B-Instruct", messages=messages, ) + + elif custom_llm_provider == "clarifai": + if "claude" in model: + return anthropic_pt(messages=messages) + elif custom_llm_provider == "perplexity": for message in messages: message.pop("name", None) diff --git a/litellm/llms/replicate.py b/litellm/llms/replicate.py index 65052e317..c29728134 100644 --- a/litellm/llms/replicate.py +++ b/litellm/llms/replicate.py @@ -1,11 +1,11 @@ import os, types import json -import requests +import requests # type: ignore import time from typing import Callable, Optional from litellm.utils import ModelResponse, Usage import litellm -import httpx +import httpx # type: ignore from .prompt_templates.factory import prompt_factory, custom_prompt diff --git a/litellm/llms/sagemaker.py b/litellm/llms/sagemaker.py index 27d3ff72a..8e75428bb 100644 --- a/litellm/llms/sagemaker.py +++ b/litellm/llms/sagemaker.py @@ -1,14 +1,14 @@ import os, types, traceback from enum import Enum import json -import requests +import requests # type: ignore import time from typing import Callable, Optional, Any import litellm from litellm.utils import ModelResponse, EmbeddingResponse, get_secret, Usage import sys from copy import deepcopy -import httpx +import httpx # type: ignore from .prompt_templates.factory import prompt_factory, custom_prompt @@ -295,7 +295,7 @@ def completion( EndpointName={model}, InferenceComponentName={model_id}, ContentType="application/json", - Body={data}, + Body={data}, # type: ignore CustomAttributes="accept_eula=true", ) """ # type: ignore @@ -321,7 +321,7 @@ def completion( response = client.invoke_endpoint( EndpointName={model}, ContentType="application/json", - Body={data}, + Body={data}, # type: ignore CustomAttributes="accept_eula=true", ) """ # type: ignore @@ -688,7 +688,7 @@ def embedding( response = client.invoke_endpoint( EndpointName={model}, ContentType="application/json", - Body={data}, + Body={data}, # type: ignore CustomAttributes="accept_eula=true", )""" # type: ignore logging_obj.pre_call( diff --git a/litellm/llms/together_ai.py b/litellm/llms/together_ai.py index 3f9d3b9de..47453ca88 100644 --- a/litellm/llms/together_ai.py +++ b/litellm/llms/together_ai.py @@ -6,11 +6,11 @@ Reference: https://docs.together.ai/docs/openai-api-compatibility import os, types import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional import litellm -import httpx +import httpx # type: ignore from litellm.utils import ModelResponse, Usage from .prompt_templates.factory import prompt_factory, custom_prompt diff --git a/litellm/llms/triton.py b/litellm/llms/triton.py new file mode 100644 index 000000000..711186b3f --- /dev/null +++ b/litellm/llms/triton.py @@ -0,0 +1,119 @@ +import os, types +import json +from enum import Enum +import requests, copy # type: ignore +import time +from typing import Callable, Optional, List +from litellm.utils import ModelResponse, Usage, map_finish_reason, CustomStreamWrapper +import litellm +from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from .base import BaseLLM +import httpx # type: ignore + + +class TritonError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", + url="https://api.anthropic.com/v1/messages", # using anthropic api base since httpx requires a url + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +class TritonChatCompletion(BaseLLM): + def __init__(self) -> None: + super().__init__() + + async def aembedding( + self, + data: dict, + model_response: litellm.utils.EmbeddingResponse, + api_base: str, + logging_obj=None, + api_key: Optional[str] = None, + ): + + async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + + response = await async_handler.post(url=api_base, data=json.dumps(data)) + + if response.status_code != 200: + raise TritonError(status_code=response.status_code, message=response.text) + + _text_response = response.text + + logging_obj.post_call(original_response=_text_response) + + _json_response = response.json() + + _outputs = _json_response["outputs"] + _output_data = _outputs[0]["data"] + _embedding_output = { + "object": "embedding", + "index": 0, + "embedding": _output_data, + } + + model_response.model = _json_response.get("model_name", "None") + model_response.data = [_embedding_output] + + return model_response + + def embedding( + self, + model: str, + input: list, + timeout: float, + api_base: str, + model_response: litellm.utils.EmbeddingResponse, + api_key: Optional[str] = None, + logging_obj=None, + optional_params=None, + client=None, + aembedding=None, + ): + data_for_triton = { + "inputs": [ + { + "name": "input_text", + "shape": [1], + "datatype": "BYTES", + "data": input, + } + ] + } + + ## LOGGING + + curl_string = f"curl {api_base} -X POST -H 'Content-Type: application/json' -d '{data_for_triton}'" + + logging_obj.pre_call( + input="", + api_key=None, + additional_args={ + "complete_input_dict": optional_params, + "request_str": curl_string, + }, + ) + + if aembedding == True: + response = self.aembedding( + data=data_for_triton, + model_response=model_response, + logging_obj=logging_obj, + api_base=api_base, + api_key=api_key, + ) + return response + else: + raise Exception( + "Only async embedding supported for triton, please use litellm.aembedding() for now" + ) diff --git a/litellm/llms/vertex_ai.py b/litellm/llms/vertex_ai.py index ce0ccc73a..84fec734f 100644 --- a/litellm/llms/vertex_ai.py +++ b/litellm/llms/vertex_ai.py @@ -1,12 +1,12 @@ import os, types import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional, Union, List from litellm.utils import ModelResponse, Usage, CustomStreamWrapper, map_finish_reason import litellm, uuid -import httpx, inspect +import httpx, inspect # type: ignore class VertexAIError(Exception): @@ -198,6 +198,23 @@ class VertexAIConfig: optional_params[mapped_params[param]] = value return optional_params + def get_eu_regions(self) -> List[str]: + """ + Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#available-regions + """ + return [ + "europe-central2", + "europe-north1", + "europe-southwest1", + "europe-west1", + "europe-west2", + "europe-west3", + "europe-west4", + "europe-west6", + "europe-west8", + "europe-west9", + ] + import asyncio @@ -419,6 +436,7 @@ def completion( from google.protobuf.struct_pb2 import Value # type: ignore from google.cloud.aiplatform_v1beta1.types import content as gapic_content_types # type: ignore import google.auth # type: ignore + import proto # type: ignore ## Load credentials with the correct quota project ref: https://github.com/googleapis/python-aiplatform/issues/2557#issuecomment-1709284744 print_verbose( @@ -605,9 +623,21 @@ def completion( ): function_call = response.candidates[0].content.parts[0].function_call args_dict = {} - for k, v in function_call.args.items(): - args_dict[k] = v - args_str = json.dumps(args_dict) + + # Check if it's a RepeatedComposite instance + for key, val in function_call.args.items(): + if isinstance( + val, proto.marshal.collections.repeated.RepeatedComposite + ): + # If so, convert to list + args_dict[key] = [v for v in val] + else: + args_dict[key] = val + + try: + args_str = json.dumps(args_dict) + except Exception as e: + raise VertexAIError(status_code=422, message=str(e)) message = litellm.Message( content=None, tool_calls=[ @@ -810,6 +840,8 @@ def completion( setattr(model_response, "usage", usage) return model_response except Exception as e: + if isinstance(e, VertexAIError): + raise e raise VertexAIError(status_code=500, message=str(e)) @@ -835,6 +867,8 @@ async def async_completion( Add support for acompletion calls for gemini-pro """ try: + import proto # type: ignore + if mode == "vision": print_verbose("\nMaking VertexAI Gemini Pro/Vision Call") print_verbose(f"\nProcessing input messages = {messages}") @@ -869,9 +903,21 @@ async def async_completion( ): function_call = response.candidates[0].content.parts[0].function_call args_dict = {} - for k, v in function_call.args.items(): - args_dict[k] = v - args_str = json.dumps(args_dict) + + # Check if it's a RepeatedComposite instance + for key, val in function_call.args.items(): + if isinstance( + val, proto.marshal.collections.repeated.RepeatedComposite + ): + # If so, convert to list + args_dict[key] = [v for v in val] + else: + args_dict[key] = val + + try: + args_str = json.dumps(args_dict) + except Exception as e: + raise VertexAIError(status_code=422, message=str(e)) message = litellm.Message( content=None, tool_calls=[ diff --git a/litellm/llms/vertex_ai_anthropic.py b/litellm/llms/vertex_ai_anthropic.py index e73545f99..3bdcf4fd6 100644 --- a/litellm/llms/vertex_ai_anthropic.py +++ b/litellm/llms/vertex_ai_anthropic.py @@ -3,7 +3,7 @@ import os, types import json from enum import Enum -import requests, copy +import requests, copy # type: ignore import time, uuid from typing import Callable, Optional, List from litellm.utils import ModelResponse, Usage, map_finish_reason, CustomStreamWrapper @@ -17,7 +17,7 @@ from .prompt_templates.factory import ( extract_between_tags, parse_xml_params, ) -import httpx +import httpx # type: ignore class VertexAIError(Exception): diff --git a/litellm/llms/vllm.py b/litellm/llms/vllm.py index 15f18cbdc..b2a9dd54d 100644 --- a/litellm/llms/vllm.py +++ b/litellm/llms/vllm.py @@ -1,8 +1,8 @@ import os import json from enum import Enum -import requests -import time, httpx +import requests # type: ignore +import time, httpx # type: ignore from typing import Callable, Any from litellm.utils import ModelResponse, Usage from .prompt_templates.factory import prompt_factory, custom_prompt diff --git a/litellm/llms/watsonx.py b/litellm/llms/watsonx.py index ac38a2a8f..34176a23a 100644 --- a/litellm/llms/watsonx.py +++ b/litellm/llms/watsonx.py @@ -1,12 +1,26 @@ from enum import Enum import json, types, time # noqa: E401 -from contextlib import contextmanager -from typing import Callable, Dict, Optional, Any, Union, List +from contextlib import asynccontextmanager, contextmanager +from typing import ( + Callable, + Dict, + Generator, + AsyncGenerator, + Iterator, + AsyncIterator, + Optional, + Any, + Union, + List, + ContextManager, + AsyncContextManager, +) -import httpx -import requests +import httpx # type: ignore +import requests # type: ignore import litellm -from litellm.utils import ModelResponse, get_secret, Usage +from litellm.utils import ModelResponse, Usage, get_secret +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from .base import BaseLLM from .prompt_templates import factory as ptf @@ -149,6 +163,15 @@ class IBMWatsonXAIConfig: optional_params[mapped_params[param]] = value return optional_params + def get_eu_regions(self) -> List[str]: + """ + Source: https://www.ibm.com/docs/en/watsonx/saas?topic=integrations-regional-availability + """ + return [ + "eu-de", + "eu-gb", + ] + def convert_messages_to_prompt(model, messages, provider, custom_prompt_dict): # handle anthropic prompts and amazon titan prompts @@ -188,11 +211,12 @@ class WatsonXAIEndpoint(str, Enum): ) EMBEDDINGS = "/ml/v1/text/embeddings" PROMPTS = "/ml/v1/prompts" + AVAILABLE_MODELS = "/ml/v1/foundation_model_specs" class IBMWatsonXAI(BaseLLM): """ - Class to interface with IBM Watsonx.ai API for text generation and embeddings. + Class to interface with IBM watsonx.ai API for text generation and embeddings. Reference: https://cloud.ibm.com/apidocs/watsonx-ai """ @@ -343,7 +367,7 @@ class IBMWatsonXAI(BaseLLM): ) if token is None and api_key is not None: # generate the auth token - if print_verbose: + if print_verbose is not None: print_verbose("Generating IAM token for Watsonx.ai") token = self.generate_iam_token(api_key) elif token is None and api_key is None: @@ -378,10 +402,11 @@ class IBMWatsonXAI(BaseLLM): print_verbose: Callable, encoding, logging_obj, - optional_params: dict, - litellm_params: Optional[dict] = None, + optional_params=None, + acompletion=None, + litellm_params=None, logger_fn=None, - timeout: Optional[float] = None, + timeout=None, ): """ Send a text generation request to the IBM Watsonx.ai API. @@ -402,12 +427,12 @@ class IBMWatsonXAI(BaseLLM): model, messages, provider, custom_prompt_dict ) - def process_text_request(request_params: dict) -> ModelResponse: - with self._manage_response( - request_params, logging_obj=logging_obj, input=prompt, timeout=timeout - ) as resp: - json_resp = resp.json() - + def process_text_gen_response(json_resp: dict) -> ModelResponse: + if "results" not in json_resp: + raise WatsonXAIError( + status_code=500, + message=f"Error: Invalid response from Watsonx.ai API: {json_resp}", + ) generated_text = json_resp["results"][0]["generated_text"] prompt_tokens = json_resp["results"][0]["input_token_count"] completion_tokens = json_resp["results"][0]["generated_token_count"] @@ -415,36 +440,70 @@ class IBMWatsonXAI(BaseLLM): model_response["finish_reason"] = json_resp["results"][0]["stop_reason"] model_response["created"] = int(time.time()) model_response["model"] = model - setattr( - model_response, - "usage", - Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ), + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, ) + setattr(model_response, "usage", usage) return model_response - def process_stream_request( - request_params: dict, + def process_stream_response( + stream_resp: Union[Iterator[str], AsyncIterator], ) -> litellm.CustomStreamWrapper: + streamwrapper = litellm.CustomStreamWrapper( + stream_resp, + model=model, + custom_llm_provider="watsonx", + logging_obj=logging_obj, + ) + return streamwrapper + + # create the function to manage the request to watsonx.ai + self.request_manager = RequestManager(logging_obj) + + def handle_text_request(request_params: dict) -> ModelResponse: + with self.request_manager.request( + request_params, + input=prompt, + timeout=timeout, + ) as resp: + json_resp = resp.json() + + return process_text_gen_response(json_resp) + + async def handle_text_request_async(request_params: dict) -> ModelResponse: + async with self.request_manager.async_request( + request_params, + input=prompt, + timeout=timeout, + ) as resp: + json_resp = resp.json() + return process_text_gen_response(json_resp) + + def handle_stream_request(request_params: dict) -> litellm.CustomStreamWrapper: # stream the response - generated chunks will be handled # by litellm.utils.CustomStreamWrapper.handle_watsonx_stream - with self._manage_response( + with self.request_manager.request( request_params, - logging_obj=logging_obj, stream=True, input=prompt, timeout=timeout, ) as resp: - response = litellm.CustomStreamWrapper( - resp.iter_lines(), - model=model, - custom_llm_provider="watsonx", - logging_obj=logging_obj, - ) - return response + streamwrapper = process_stream_response(resp.iter_lines()) + return streamwrapper + + async def handle_stream_request_async(request_params: dict) -> litellm.CustomStreamWrapper: + # stream the response - generated chunks will be handled + # by litellm.utils.CustomStreamWrapper.handle_watsonx_stream + async with self.request_manager.async_request( + request_params, + stream=True, + input=prompt, + timeout=timeout, + ) as resp: + streamwrapper = process_stream_response(resp.aiter_lines()) + return streamwrapper try: ## Get the response from the model @@ -455,10 +514,18 @@ class IBMWatsonXAI(BaseLLM): optional_params=optional_params, print_verbose=print_verbose, ) - if stream: - return process_stream_request(req_params) + if stream and (acompletion is True): + # stream and async text generation + return handle_stream_request_async(req_params) + elif stream: + # streaming text generation + return handle_stream_request(req_params) + elif (acompletion is True): + # async text generation + return handle_text_request_async(req_params) else: - return process_text_request(req_params) + # regular text generation + return handle_text_request(req_params) except WatsonXAIError as e: raise e except Exception as e: @@ -473,6 +540,7 @@ class IBMWatsonXAI(BaseLLM): model_response=None, optional_params=None, encoding=None, + aembedding=None, ): """ Send a text embedding request to the IBM Watsonx.ai API. @@ -507,9 +575,6 @@ class IBMWatsonXAI(BaseLLM): } request_params = dict(version=api_params["api_version"]) url = api_params["url"].rstrip("/") + WatsonXAIEndpoint.EMBEDDINGS - # request = httpx.Request( - # "POST", url, headers=headers, json=payload, params=request_params - # ) req_params = { "method": "POST", "url": url, @@ -517,25 +582,49 @@ class IBMWatsonXAI(BaseLLM): "json": payload, "params": request_params, } - with self._manage_response( - req_params, logging_obj=logging_obj, input=input - ) as resp: - json_resp = resp.json() + request_manager = RequestManager(logging_obj) - results = json_resp.get("results", []) - embedding_response = [] - for idx, result in enumerate(results): - embedding_response.append( - {"object": "embedding", "index": idx, "embedding": result["embedding"]} + def process_embedding_response(json_resp: dict) -> ModelResponse: + results = json_resp.get("results", []) + embedding_response = [] + for idx, result in enumerate(results): + embedding_response.append( + { + "object": "embedding", + "index": idx, + "embedding": result["embedding"], + } + ) + model_response["object"] = "list" + model_response["data"] = embedding_response + model_response["model"] = model + input_tokens = json_resp.get("input_token_count", 0) + model_response.usage = Usage( + prompt_tokens=input_tokens, + completion_tokens=0, + total_tokens=input_tokens, ) - model_response["object"] = "list" - model_response["data"] = embedding_response - model_response["model"] = model - input_tokens = json_resp.get("input_token_count", 0) - model_response.usage = Usage( - prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens - ) - return model_response + return model_response + + def handle_embedding(request_params: dict) -> ModelResponse: + with request_manager.request(request_params, input=input) as resp: + json_resp = resp.json() + return process_embedding_response(json_resp) + + async def handle_aembedding(request_params: dict) -> ModelResponse: + async with request_manager.async_request(request_params, input=input) as resp: + json_resp = resp.json() + return process_embedding_response(json_resp) + + try: + if aembedding is True: + return handle_embedding(req_params) + else: + return handle_aembedding(req_params) + except WatsonXAIError as e: + raise e + except Exception as e: + raise WatsonXAIError(status_code=500, message=str(e)) def generate_iam_token(self, api_key=None, **params): headers = {} @@ -558,52 +647,144 @@ class IBMWatsonXAI(BaseLLM): self.token = iam_access_token return iam_access_token - @contextmanager - def _manage_response( - self, - request_params: dict, - logging_obj: Any, - stream: bool = False, - input: Optional[Any] = None, - timeout: Optional[float] = None, - ): - request_str = ( - f"response = {request_params['method']}(\n" - f"\turl={request_params['url']},\n" - f"\tjson={request_params['json']},\n" - f")" - ) - logging_obj.pre_call( - input=input, - api_key=request_params["headers"].get("Authorization"), - additional_args={ - "complete_input_dict": request_params["json"], - "request_str": request_str, - }, - ) - if timeout: - request_params["timeout"] = timeout - try: - if stream: - resp = requests.request( - **request_params, - stream=True, - ) - resp.raise_for_status() - yield resp - else: - resp = requests.request(**request_params) - resp.raise_for_status() - yield resp - except Exception as e: - raise WatsonXAIError(status_code=500, message=str(e)) - if not stream: - logging_obj.post_call( + def get_available_models(self, *, ids_only: bool = True, **params): + api_params = self._get_api_params(params) + headers = { + "Authorization": f"Bearer {api_params['token']}", + "Content-Type": "application/json", + "Accept": "application/json", + } + request_params = dict(version=api_params["api_version"]) + url = api_params["url"].rstrip("/") + WatsonXAIEndpoint.AVAILABLE_MODELS + req_params = dict(method="GET", url=url, headers=headers, params=request_params) + with RequestManager(logging_obj=None).request(req_params) as resp: + json_resp = resp.json() + if not ids_only: + return json_resp + return [res["model_id"] for res in json_resp["resources"]] + +class RequestManager: + """ + Returns a context manager that manages the response from the request. + if async_ is True, returns an async context manager, otherwise returns a regular context manager. + + Usage: + ```python + request_params = dict(method="POST", url="https://api.example.com", headers={"Authorization" : "Bearer token"}, json={"key": "value"}) + request_manager = RequestManager(logging_obj=logging_obj) + async with request_manager.request(request_params) as resp: + ... + # or + with request_manager.async_request(request_params) as resp: + ... + ``` + """ + + def __init__(self, logging_obj=None): + self.logging_obj = logging_obj + + def pre_call( + self, + request_params: dict, + input: Optional[Any] = None, + ): + if self.logging_obj is None: + return + request_str = ( + f"response = {request_params['method']}(\n" + f"\turl={request_params['url']},\n" + f"\tjson={request_params.get('json')},\n" + f")" + ) + self.logging_obj.pre_call( + input=input, + api_key=request_params["headers"].get("Authorization"), + additional_args={ + "complete_input_dict": request_params.get("json"), + "request_str": request_str, + }, + ) + + def post_call(self, resp, request_params): + if self.logging_obj is None: + return + self.logging_obj.post_call( input=input, api_key=request_params["headers"].get("Authorization"), original_response=json.dumps(resp.json()), additional_args={ "status_code": resp.status_code, - "complete_input_dict": request_params["json"], + "complete_input_dict": request_params.get( + "data", request_params.get("json") + ), }, ) + + @contextmanager + def request( + self, + request_params: dict, + stream: bool = False, + input: Optional[Any] = None, + timeout=None, + ) -> Generator[requests.Response, None, None]: + """ + Returns a context manager that yields the response from the request. + """ + self.pre_call(request_params, input) + if timeout: + request_params["timeout"] = timeout + if stream: + request_params["stream"] = stream + try: + resp = requests.request(**request_params) + if not resp.ok: + raise WatsonXAIError( + status_code=resp.status_code, + message=f"Error {resp.status_code} ({resp.reason}): {resp.text}", + ) + yield resp + except Exception as e: + raise WatsonXAIError(status_code=500, message=str(e)) + if not stream: + self.post_call(resp, request_params) + + @asynccontextmanager + async def async_request( + self, + request_params: dict, + stream: bool = False, + input: Optional[Any] = None, + timeout=None, + ) -> AsyncGenerator[httpx.Response, None]: + self.pre_call(request_params, input) + if timeout: + request_params["timeout"] = timeout + if stream: + request_params["stream"] = stream + try: + # async with AsyncHTTPHandler(timeout=timeout) as client: + self.async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout( + timeout=request_params.pop("timeout", 600.0), connect=5.0 + ), + ) + # async_handler.client.verify = False + if "json" in request_params: + request_params["data"] = json.dumps(request_params.pop("json", {})) + method = request_params.pop("method") + if method.upper() == "POST": + resp = await self.async_handler.post(**request_params) + else: + resp = await self.async_handler.get(**request_params) + if resp.status_code not in [200, 201]: + raise WatsonXAIError( + status_code=resp.status_code, + message=f"Error {resp.status_code} ({resp.reason}): {resp.text}", + ) + yield resp + # await async_handler.close() + except Exception as e: + raise WatsonXAIError(status_code=500, message=str(e)) + if not stream: + self.post_call(resp, request_params) \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 9765669fe..6156d9c39 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -9,12 +9,12 @@ import os, openai, sys, json, inspect, uuid, datetime, threading from typing import Any, Literal, Union, BinaryIO +from typing_extensions import overload from functools import partial import dotenv, traceback, random, asyncio, time, contextvars from copy import deepcopy import httpx import litellm - from ._logging import verbose_logger from litellm import ( # type: ignore client, @@ -39,6 +39,7 @@ from litellm.utils import ( Usage, get_optional_params_embeddings, get_optional_params_image_gen, + supports_httpx_timeout, ) from .llms import ( anthropic_text, @@ -46,6 +47,7 @@ from .llms import ( ai21, sagemaker, bedrock, + triton, huggingface_restapi, replicate, aleph_alpha, @@ -55,6 +57,7 @@ from .llms import ( ollama, ollama_chat, cloudflare, + clarifai, cohere, cohere_chat, petals, @@ -73,10 +76,14 @@ from .llms.azure_text import AzureTextCompletion from .llms.anthropic import AnthropicChatCompletion from .llms.anthropic_text import AnthropicTextCompletion from .llms.huggingface_restapi import Huggingface +from .llms.predibase import PredibaseChatCompletion +from .llms.bedrock_httpx import BedrockLLM +from .llms.triton import TritonChatCompletion from .llms.prompt_templates.factory import ( prompt_factory, custom_prompt, function_call_prompt, + map_system_message_pt, ) import tiktoken from concurrent.futures import ThreadPoolExecutor @@ -100,7 +107,6 @@ from litellm.utils import ( ) ####### ENVIRONMENT VARIABLES ################### -dotenv.load_dotenv() # Loading env variables using dotenv openai_chat_completions = OpenAIChatCompletion() openai_text_completions = OpenAITextCompletion() anthropic_chat_completions = AnthropicChatCompletion() @@ -108,6 +114,9 @@ anthropic_text_completions = AnthropicTextCompletion() azure_chat_completions = AzureChatCompletion() azure_text_completions = AzureTextCompletion() huggingface = Huggingface() +predibase_chat_completions = PredibaseChatCompletion() +triton_chat_completions = TritonChatCompletion() +bedrock_chat_completion = BedrockLLM() ####### COMPLETION ENDPOINTS ################ @@ -186,6 +195,7 @@ async def acompletion( top_p: Optional[float] = None, n: Optional[int] = None, stream: Optional[bool] = None, + stream_options: Optional[dict] = None, stop=None, max_tokens: Optional[int] = None, presence_penalty: Optional[float] = None, @@ -205,6 +215,7 @@ async def acompletion( api_version: Optional[str] = None, api_key: Optional[str] = None, model_list: Optional[list] = None, # pass in a list of api_base,keys, etc. + extra_headers: Optional[dict] = None, # Optional liteLLM function params **kwargs, ): @@ -222,6 +233,7 @@ async def acompletion( top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0). n (int, optional): The number of completions to generate (default is 1). stream (bool, optional): If True, return a streaming response (default is False). + stream_options (dict, optional): A dictionary containing options for the streaming response. Only use this if stream is True. stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens. max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity). presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far. @@ -247,7 +259,7 @@ async def acompletion( - If `stream` is True, the function returns an async generator that yields completion lines. """ loop = asyncio.get_event_loop() - custom_llm_provider = None + custom_llm_provider = kwargs.get("custom_llm_provider", None) # Adjusted to use explicit arguments instead of *args and **kwargs completion_kwargs = { "model": model, @@ -259,6 +271,7 @@ async def acompletion( "top_p": top_p, "n": n, "stream": stream, + "stream_options": stream_options, "stop": stop, "max_tokens": max_tokens, "presence_penalty": presence_penalty, @@ -278,9 +291,10 @@ async def acompletion( "model_list": model_list, "acompletion": True, # assuming this is a required parameter } - _, custom_llm_provider, _, _ = get_llm_provider( - model=model, api_base=completion_kwargs.get("base_url", None) - ) + if custom_llm_provider is None: + _, custom_llm_provider, _, _ = get_llm_provider( + model=model, api_base=completion_kwargs.get("base_url", None) + ) try: # Use a partial function to pass your keyword arguments func = partial(completion, **completion_kwargs, **kwargs) @@ -289,9 +303,6 @@ async def acompletion( ctx = contextvars.copy_context() func_with_context = partial(ctx.run, func) - _, custom_llm_provider, _, _ = get_llm_provider( - model=model, api_base=kwargs.get("api_base", None) - ) if ( custom_llm_provider == "openai" or custom_llm_provider == "azure" @@ -303,6 +314,7 @@ async def acompletion( or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "deepseek" or custom_llm_provider == "text-completion-openai" or custom_llm_provider == "huggingface" or custom_llm_provider == "ollama" @@ -311,6 +323,8 @@ async def acompletion( or custom_llm_provider == "gemini" or custom_llm_provider == "sagemaker" or custom_llm_provider == "anthropic" + or custom_llm_provider == "predibase" + or (custom_llm_provider == "bedrock" and "cohere" in model) or custom_llm_provider in litellm.openai_compatible_providers ): # currently implemented aiohttp calls for just azure, openai, hf, ollama, vertex ai soon all. init_response = await loop.run_in_executor(None, func_with_context) @@ -450,11 +464,12 @@ def completion( model: str, # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create messages: List = [], - timeout: Optional[Union[float, int]] = None, + timeout: Optional[Union[float, str, httpx.Timeout]] = None, temperature: Optional[float] = None, top_p: Optional[float] = None, n: Optional[int] = None, stream: Optional[bool] = None, + stream_options: Optional[dict] = None, stop=None, max_tokens: Optional[int] = None, presence_penalty: Optional[float] = None, @@ -494,6 +509,7 @@ def completion( top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0). n (int, optional): The number of completions to generate (default is 1). stream (bool, optional): If True, return a streaming response (default is False). + stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true. stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens. max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity). presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far. @@ -553,6 +569,7 @@ def completion( eos_token = kwargs.get("eos_token", None) preset_cache_key = kwargs.get("preset_cache_key", None) hf_model_name = kwargs.get("hf_model_name", None) + supports_system_message = kwargs.get("supports_system_message", None) ### TEXT COMPLETION CALLS ### text_completion = kwargs.get("text_completion", False) atext_completion = kwargs.get("atext_completion", False) @@ -570,6 +587,7 @@ def completion( "top_p", "n", "stream", + "stream_options", "stop", "max_tokens", "presence_penalty", @@ -618,6 +636,7 @@ def completion( "model_list", "num_retries", "context_window_fallback_dict", + "retry_policy", "roles", "final_prompt_value", "bos_token", @@ -643,16 +662,30 @@ def completion( "no-log", "base_model", "stream_timeout", + "supports_system_message", + "region_name", + "allowed_model_region", ] + default_params = openai_params + litellm_params non_default_params = { k: v for k, v in kwargs.items() if k not in default_params } # model-specific params - pass them straight to the model/provider - if timeout is None: - timeout = ( - kwargs.get("request_timeout", None) or 600 - ) # set timeout for 10 minutes by default - timeout = float(timeout) + + ### TIMEOUT LOGIC ### + timeout = timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + try: if base_url is not None: api_base = base_url @@ -694,7 +727,6 @@ def completion( ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### if input_cost_per_token is not None and output_cost_per_token is not None: - print_verbose(f"Registering model={model} in model cost map") litellm.register_model( { f"{custom_llm_provider}/{model}": { @@ -747,6 +779,13 @@ def completion( custom_prompt_dict[model]["bos_token"] = bos_token if eos_token: custom_prompt_dict[model]["eos_token"] = eos_token + + if ( + supports_system_message is not None + and isinstance(supports_system_message, bool) + and supports_system_message == False + ): + messages = map_system_message_pt(messages=messages) model_api_key = get_api_key( llm_provider=custom_llm_provider, dynamic_api_key=api_key ) # get the api key from the environment if required for the model @@ -761,6 +800,7 @@ def completion( top_p=top_p, n=n, stream=stream, + stream_options=stream_options, stop=stop, max_tokens=max_tokens, presence_penalty=presence_penalty, @@ -808,6 +848,10 @@ def completion( proxy_server_request=proxy_server_request, preset_cache_key=preset_cache_key, no_log=no_log, + input_cost_per_second=input_cost_per_second, + input_cost_per_token=input_cost_per_token, + output_cost_per_second=output_cost_per_second, + output_cost_per_token=output_cost_per_token, ) logging.update_environment_variables( model=model, @@ -873,7 +917,7 @@ def completion( logger_fn=logger_fn, logging_obj=logging, acompletion=acompletion, - timeout=timeout, + timeout=timeout, # type: ignore client=client, # pass AsyncAzureOpenAI, AzureOpenAI client ) @@ -960,6 +1004,7 @@ def completion( or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "deepseek" or custom_llm_provider == "anyscale" or custom_llm_provider == "mistral" or custom_llm_provider == "openai" @@ -1014,7 +1059,7 @@ def completion( optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - timeout=timeout, + timeout=timeout, # type: ignore custom_prompt_dict=custom_prompt_dict, client=client, # pass AsyncOpenAI, OpenAI client organization=organization, @@ -1099,7 +1144,7 @@ def completion( optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn, - timeout=timeout, + timeout=timeout, # type: ignore ) if ( @@ -1172,6 +1217,61 @@ def completion( ) response = model_response + elif ( + "clarifai" in model + or custom_llm_provider == "clarifai" + or model in litellm.clarifai_models + ): + clarifai_key = None + clarifai_key = ( + api_key + or litellm.clarifai_key + or litellm.api_key + or get_secret("CLARIFAI_API_KEY") + or get_secret("CLARIFAI_API_TOKEN") + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("CLARIFAI_API_BASE") + or "https://api.clarifai.com/v2" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = clarifai.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + acompletion=acompletion, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=clarifai_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=model_response, + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=clarifai_key, + original_response=model_response, + ) + response = model_response elif custom_llm_provider == "anthropic": api_key = ( @@ -1473,7 +1573,7 @@ def completion( acompletion=acompletion, logging_obj=logging, custom_prompt_dict=custom_prompt_dict, - timeout=timeout, + timeout=timeout, # type: ignore ) if ( "stream" in optional_params @@ -1566,7 +1666,7 @@ def completion( logger_fn=logger_fn, logging_obj=logging, acompletion=acompletion, - timeout=timeout, + timeout=timeout, # type: ignore ) ## LOGGING logging.post_call( @@ -1751,6 +1851,52 @@ def completion( ) return response response = model_response + elif custom_llm_provider == "predibase": + tenant_id = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + api_base = ( + optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.predibase_key + or get_secret("PREDIBASE_API_KEY") + ) + + _model_response = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + return _model_response + response = _model_response elif custom_llm_provider == "ai21": custom_llm_provider = "ai21" ai21_key = ( @@ -1835,40 +1981,59 @@ def completion( elif custom_llm_provider == "bedrock": # boto3 reads keys from .env custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - response = bedrock.completion( - model=model, - messages=messages, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=encoding, - logging_obj=logging, - timeout=timeout, - ) - if ( - "stream" in optional_params - and optional_params["stream"] == True - and not isinstance(response, CustomStreamWrapper) - ): - # don't try to access stream object, - if "ai21" in model: - response = CustomStreamWrapper( - response, - model, - custom_llm_provider="bedrock", - logging_obj=logging, - ) - else: - response = CustomStreamWrapper( - iter(response), - model, - custom_llm_provider="bedrock", - logging_obj=logging, - ) + if "cohere" in model: + response = bedrock_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + acompletion=acompletion, + ) + else: + response = bedrock.completion( + model=model, + messages=messages, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and not isinstance(response, CustomStreamWrapper) + ): + # don't try to access stream object, + if "ai21" in model: + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="bedrock", + logging_obj=logging, + ) + else: + response = CustomStreamWrapper( + iter(response), + model, + custom_llm_provider="bedrock", + logging_obj=logging, + ) if optional_params.get("stream", False): ## LOGGING @@ -1893,7 +2058,7 @@ def completion( logger_fn=logger_fn, encoding=encoding, logging_obj=logging, - timeout=timeout, + timeout=timeout, # type: ignore ) if ( "stream" in optional_params @@ -2145,7 +2310,7 @@ def completion( """ assume input to custom LLM api bases follow this format: resp = requests.post( - api_base, + api_base, json={ 'model': 'meta-llama/Llama-2-13b-hf', # model name 'params': { @@ -2274,7 +2439,7 @@ def batch_completion( n: Optional[int] = None, stream: Optional[bool] = None, stop=None, - max_tokens: Optional[float] = None, + max_tokens: Optional[int] = None, presence_penalty: Optional[float] = None, frequency_penalty: Optional[float] = None, logit_bias: Optional[dict] = None, @@ -2537,11 +2702,13 @@ async def aembedding(*args, **kwargs): or custom_llm_provider == "voyage" or custom_llm_provider == "mistral" or custom_llm_provider == "custom_openai" + or custom_llm_provider == "triton" or custom_llm_provider == "anyscale" or custom_llm_provider == "openrouter" or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "deepseek" or custom_llm_provider == "fireworks_ai" or custom_llm_provider == "ollama" or custom_llm_provider == "vertex_ai" @@ -2667,6 +2834,7 @@ def embedding( "model_list", "num_retries", "context_window_fallback_dict", + "retry_policy", "roles", "final_prompt_value", "bos_token", @@ -2690,6 +2858,8 @@ def embedding( "ttl", "cache", "no-log", + "region_name", + "allowed_model_region", ] default_params = openai_params + litellm_params non_default_params = { @@ -2866,23 +3036,43 @@ def embedding( optional_params=optional_params, model_response=EmbeddingResponse(), ) + elif custom_llm_provider == "triton": + if api_base is None: + raise ValueError( + "api_base is required for triton. Please pass `api_base`" + ) + response = triton_chat_completions.embedding( + model=model, + input=input, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + ) elif custom_llm_provider == "vertex_ai": vertex_ai_project = ( optional_params.pop("vertex_project", None) or optional_params.pop("vertex_ai_project", None) or litellm.vertex_project or get_secret("VERTEXAI_PROJECT") + or get_secret("VERTEX_PROJECT") ) vertex_ai_location = ( optional_params.pop("vertex_location", None) or optional_params.pop("vertex_ai_location", None) or litellm.vertex_location or get_secret("VERTEXAI_LOCATION") + or get_secret("VERTEX_LOCATION") ) vertex_credentials = ( optional_params.pop("vertex_credentials", None) or optional_params.pop("vertex_ai_credentials", None) or get_secret("VERTEXAI_CREDENTIALS") + or get_secret("VERTEX_CREDENTIALS") ) response = vertex_ai.embedding( @@ -2923,16 +3113,18 @@ def embedding( model=model, # type: ignore llm_provider="ollama", # type: ignore ) - if aembedding: - response = ollama.ollama_aembeddings( - api_base=api_base, - model=model, - prompts=input, - encoding=encoding, - logging_obj=logging, - optional_params=optional_params, - model_response=EmbeddingResponse(), - ) + ollama_embeddings_fn = ( + ollama.ollama_aembeddings if aembedding else ollama.ollama_embeddings + ) + response = ollama_embeddings_fn( + api_base=api_base, + model=model, + prompts=input, + encoding=encoding, + logging_obj=logging, + optional_params=optional_params, + model_response=EmbeddingResponse(), + ) elif custom_llm_provider == "sagemaker": response = sagemaker.embedding( model=model, @@ -3061,11 +3253,13 @@ async def atext_completion(*args, **kwargs): or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "groq" + or custom_llm_provider == "deepseek" or custom_llm_provider == "fireworks_ai" or custom_llm_provider == "text-completion-openai" or custom_llm_provider == "huggingface" or custom_llm_provider == "ollama" or custom_llm_provider == "vertex_ai" + or custom_llm_provider in litellm.openai_compatible_providers ): # currently implemented aiohttp calls for just azure and openai, soon all. # Await normally response = await loop.run_in_executor(None, func_with_context) @@ -3096,6 +3290,8 @@ async def atext_completion(*args, **kwargs): ## TRANSLATE CHAT TO TEXT FORMAT ## if isinstance(response, TextCompletionResponse): return response + elif asyncio.iscoroutine(response): + response = await response text_completion_response = TextCompletionResponse() text_completion_response["id"] = response.get("id", None) @@ -3155,6 +3351,7 @@ def text_completion( Union[str, List[str]] ] = None, # Optional: Sequences where the API will stop generating further tokens. stream: Optional[bool] = None, # Optional: Whether to stream back partial progress. + stream_options: Optional[dict] = None, suffix: Optional[ str ] = None, # Optional: The suffix that comes after a completion of inserted text. @@ -3232,6 +3429,8 @@ def text_completion( optional_params["stop"] = stop if stream is not None: optional_params["stream"] = stream + if stream_options is not None: + optional_params["stream_options"] = stream_options if suffix is not None: optional_params["suffix"] = suffix if temperature is not None: @@ -3342,7 +3541,9 @@ def text_completion( if kwargs.get("acompletion", False) == True: return response if stream == True or kwargs.get("stream", False) == True: - response = TextCompletionStreamWrapper(completion_stream=response, model=model) + response = TextCompletionStreamWrapper( + completion_stream=response, model=model, stream_options=stream_options + ) return response transformed_logprobs = None # only supported for TGI models @@ -3536,6 +3737,7 @@ def image_generation( "model_list", "num_retries", "context_window_fallback_dict", + "retry_policy", "roles", "final_prompt_value", "bos_token", @@ -3556,6 +3758,8 @@ def image_generation( "caching_groups", "ttl", "cache", + "region_name", + "allowed_model_region", ] default_params = openai_params + litellm_params non_default_params = { diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7fcd425bb..0a262e310 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9,6 +9,30 @@ "mode": "chat", "supports_function_calling": true }, + "gpt-4o": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "gpt-4o-2024-05-13": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, "gpt-4-turbo-preview": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -739,6 +763,24 @@ "litellm_provider": "mistral", "mode": "embedding" }, + "deepseek-chat": { + "max_tokens": 4096, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000014, + "output_cost_per_token": 0.00000028, + "litellm_provider": "deepseek", + "mode": "chat" + }, + "deepseek-coder": { + "max_tokens": 4096, + "max_input_tokens": 16000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.00000014, + "output_cost_per_token": 0.00000028, + "litellm_provider": "deepseek", + "mode": "chat" + }, "groq/llama2-70b-4096": { "max_tokens": 4096, "max_input_tokens": 4096, @@ -1060,8 +1102,8 @@ "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0, - "output_cost_per_token": 0, + "input_cost_per_token": 0.000000625, + "output_cost_per_token": 0.000001875, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, @@ -1072,8 +1114,8 @@ "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0, - "output_cost_per_token": 0, + "input_cost_per_token": 0.000000625, + "output_cost_per_token": 0.000001875, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, @@ -1084,8 +1126,8 @@ "max_tokens": 8192, "max_input_tokens": 1000000, "max_output_tokens": 8192, - "input_cost_per_token": 0, - "output_cost_per_token": 0, + "input_cost_per_token": 0.000000625, + "output_cost_per_token": 0.000001875, "litellm_provider": "vertex_ai-language-models", "mode": "chat", "supports_function_calling": true, @@ -1553,6 +1595,135 @@ "litellm_provider": "replicate", "mode": "chat" }, + "openrouter/microsoft/wizardlm-2-8x22b:nitro": { + "max_tokens": 65536, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/google/gemini-pro-1.5": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.0000075, + "input_cost_per_image": 0.00265, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/mistralai/mixtral-8x22b-instruct": { + "max_tokens": 65536, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000065, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/cohere/command-r-plus": { + "max_tokens": 128000, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/databricks/dbrx-instruct": { + "max_tokens": 32768, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000006, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/anthropic/claude-3-haiku": { + "max_tokens": 200000, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "input_cost_per_image": 0.0004, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/anthropic/claude-3-sonnet": { + "max_tokens": 200000, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "input_cost_per_image": 0.0048, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/mistralai/mistral-large": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { + "max_tokens": 32769, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/google/gemini-pro-vision": { + "max_tokens": 45875, + "input_cost_per_token": 0.000000125, + "output_cost_per_token": 0.000000375, + "input_cost_per_image": 0.0025, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/fireworks/firellava-13b": { + "max_tokens": 4096, + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-8b-instruct:free": { + "max_tokens": 8192, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-8b-instruct:extended": { + "max_tokens": 16384, + "input_cost_per_token": 0.000000225, + "output_cost_per_token": 0.00000225, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-70b-instruct:nitro": { + "max_tokens": 8192, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-70b-instruct": { + "max_tokens": 8192, + "input_cost_per_token": 0.00000059, + "output_cost_per_token": 0.00000079, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/openai/gpt-4-vision-preview": { + "max_tokens": 130000, + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "input_cost_per_image": 0.01445, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, "openrouter/openai/gpt-3.5-turbo": { "max_tokens": 4095, "input_cost_per_token": 0.0000015, @@ -1603,14 +1774,14 @@ "tool_use_system_prompt_tokens": 395 }, "openrouter/google/palm-2-chat-bison": { - "max_tokens": 8000, + "max_tokens": 25804, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/google/palm-2-codechat-bison": { - "max_tokens": 8000, + "max_tokens": 20070, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", @@ -1693,13 +1864,6 @@ "litellm_provider": "openrouter", "mode": "chat" }, - "openrouter/meta-llama/llama-3-70b-instruct": { - "max_tokens": 8192, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000008, - "litellm_provider": "openrouter", - "mode": "chat" - }, "j2-ultra": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -1832,6 +1996,15 @@ "litellm_provider": "bedrock", "mode": "embedding" }, + "amazon.titan-embed-text-v2:0": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "output_vector_size": 1024, + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0, + "litellm_provider": "bedrock", + "mode": "embedding" + }, "mistral.mistral-7b-instruct-v0:2": { "max_tokens": 8191, "max_input_tokens": 32000, @@ -2495,6 +2668,24 @@ "litellm_provider": "bedrock", "mode": "chat" }, + "cohere.command-r-plus-v1:0": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000030, + "output_cost_per_token": 0.000015, + "litellm_provider": "bedrock", + "mode": "chat" + }, + "cohere.command-r-v1:0": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000015, + "litellm_provider": "bedrock", + "mode": "chat" + }, "cohere.embed-english-v3": { "max_tokens": 512, "max_input_tokens": 512, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index ae30e10a8..b70559084 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1,5 +1 @@ -<<<<<<< HEAD -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

-======= -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

->>>>>>> 73a7b4f4 (refactor(main.py): trigger new build) +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-052c4579f80d66ae.js b/litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-052c4579f80d66ae.js new file mode 100644 index 000000000..030bbbe66 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-052c4579f80d66ae.js @@ -0,0 +1 @@ +"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[936],{26463:function(e,n,t){t.d(n,{HJ:function(){return ea},_W:function(){return e8}});var o,r,a,i=t(64090),l=t(68005),s=t(32633),d=t(46826),c=t(49492),u=t(85895),f=t(9861),p=t(94171),v=t(8053),h=t(47389),y=t(84120),m=t(59659),b=t(19337),x=t(7212),_=t(67990),w=t(75076),g=t(38260),j=t(3444),Z=t(13256),N=t(81131),k=t(53030),M=t(65620),D=t(45503),C=t(57324),P=t(88800),O=t(74416),S=t(50295),L=t(24592),W=t(39188),E=t(91181),I=t(642),F=t(82985),T=function(){return(T=Object.assign||function(e){for(var n,t=1,o=arguments.length;t(0,h.Z)(d,l)&&(l=(0,y.Z)(d,-1*((void 0===u?1:u)-1))),c&&0>(0,h.Z)(l,c)&&(l=c),f=(0,s.Z)(l),p=n.month,x=(v=(0,i.useState)(f))[0],_=[void 0===p?x:p,v[1]])[0],g=_[1],[w,function(e){if(!n.disableNavigation){var t,o=(0,s.Z)(e);g(o),null===(t=n.onMonthChange)||void 0===t||t.call(n,o)}}]),N=Z[0],k=Z[1],M=function(e,n){for(var t=n.reverseMonths,o=n.numberOfMonths,r=(0,s.Z)(e),a=(0,s.Z)((0,y.Z)(r,o)),i=(0,h.Z)(a,r),l=[],d=0;d=(0,h.Z)(a,t)))return(0,y.Z)(a,-(o?void 0===r?1:r:1))}}(N,j),P=function(e){return M.some(function(n){return(0,m.Z)(e,n)})};return Y.jsx(eo.Provider,{value:{currentMonth:N,displayMonths:M,goToMonth:k,goToDate:function(e,n){P(e)||(n&&(0,b.Z)(e,n)?k((0,y.Z)(e,1+-1*j.numberOfMonths)):k(e))},previousMonth:C,nextMonth:D,isDateDisplayed:P},children:e.children})}function ea(){var e=(0,i.useContext)(eo);if(!e)throw Error("useNavigation must be used within a NavigationProvider");return e}function ei(e){var n,t=Q(),o=t.classNames,r=t.styles,a=t.components,i=ea().goToMonth,l=function(n){i((0,y.Z)(n,e.displayIndex?-e.displayIndex:0))},s=null!==(n=null==a?void 0:a.CaptionLabel)&&void 0!==n?n:V,d=Y.jsx(s,{id:e.id,displayMonth:e.displayMonth});return Y.jsxs("div",{className:o.caption_dropdowns,style:r.caption_dropdowns,children:[Y.jsx("div",{className:o.vhidden,children:d}),Y.jsx(en,{onChange:l,displayMonth:e.displayMonth}),Y.jsx(et,{onChange:l,displayMonth:e.displayMonth})]})}function el(e){return Y.jsx("svg",T({width:"16px",height:"16px",viewBox:"0 0 120 120"},e,{children:Y.jsx("path",{d:"M69.490332,3.34314575 C72.6145263,0.218951416 77.6798462,0.218951416 80.8040405,3.34314575 C83.8617626,6.40086786 83.9268205,11.3179931 80.9992143,14.4548388 L80.8040405,14.6568542 L35.461,60 L80.8040405,105.343146 C83.8617626,108.400868 83.9268205,113.317993 80.9992143,116.454839 L80.8040405,116.656854 C77.7463184,119.714576 72.8291931,119.779634 69.6923475,116.852028 L69.490332,116.656854 L18.490332,65.6568542 C15.4326099,62.5991321 15.367552,57.6820069 18.2951583,54.5451612 L18.490332,54.3431458 L69.490332,3.34314575 Z",fill:"currentColor",fillRule:"nonzero"})}))}function es(e){return Y.jsx("svg",T({width:"16px",height:"16px",viewBox:"0 0 120 120"},e,{children:Y.jsx("path",{d:"M49.8040405,3.34314575 C46.6798462,0.218951416 41.6145263,0.218951416 38.490332,3.34314575 C35.4326099,6.40086786 35.367552,11.3179931 38.2951583,14.4548388 L38.490332,14.6568542 L83.8333725,60 L38.490332,105.343146 C35.4326099,108.400868 35.367552,113.317993 38.2951583,116.454839 L38.490332,116.656854 C41.5480541,119.714576 46.4651794,119.779634 49.602025,116.852028 L49.8040405,116.656854 L100.804041,65.6568542 C103.861763,62.5991321 103.926821,57.6820069 100.999214,54.5451612 L100.804041,54.3431458 L49.8040405,3.34314575 Z",fill:"currentColor"})}))}var ed=(0,i.forwardRef)(function(e,n){var t=Q(),o=t.classNames,r=t.styles,a=[o.button_reset,o.button];e.className&&a.push(e.className);var i=a.join(" "),l=T(T({},r.button_reset),r.button);return e.style&&Object.assign(l,e.style),Y.jsx("button",T({},e,{ref:n,type:"button",className:i,style:l}))});function ec(e){var n,t,o=Q(),r=o.dir,a=o.locale,i=o.classNames,l=o.styles,s=o.labels,d=s.labelPrevious,c=s.labelNext,u=o.components;if(!e.nextMonth&&!e.previousMonth)return Y.jsx(Y.Fragment,{});var f=d(e.previousMonth,{locale:a}),p=[i.nav_button,i.nav_button_previous].join(" "),v=c(e.nextMonth,{locale:a}),h=[i.nav_button,i.nav_button_next].join(" "),y=null!==(n=null==u?void 0:u.IconRight)&&void 0!==n?n:es,m=null!==(t=null==u?void 0:u.IconLeft)&&void 0!==t?t:el;return Y.jsxs("div",{className:i.nav,style:l.nav,children:[!e.hidePrevious&&Y.jsx(ed,{name:"previous-month","aria-label":f,className:p,style:l.nav_button_previous,disabled:!e.previousMonth,onClick:e.onPreviousClick,children:"rtl"===r?Y.jsx(y,{className:i.nav_icon,style:l.nav_icon}):Y.jsx(m,{className:i.nav_icon,style:l.nav_icon})}),!e.hideNext&&Y.jsx(ed,{name:"next-month","aria-label":v,className:h,style:l.nav_button_next,disabled:!e.nextMonth,onClick:e.onNextClick,children:"rtl"===r?Y.jsx(m,{className:i.nav_icon,style:l.nav_icon}):Y.jsx(y,{className:i.nav_icon,style:l.nav_icon})})]})}function eu(e){var n=Q().numberOfMonths,t=ea(),o=t.previousMonth,r=t.nextMonth,a=t.goToMonth,i=t.displayMonths,l=i.findIndex(function(n){return(0,m.Z)(e.displayMonth,n)}),s=0===l,d=l===i.length-1;return Y.jsx(ec,{displayMonth:e.displayMonth,hideNext:n>1&&(s||!d),hidePrevious:n>1&&(d||!s),nextMonth:r,previousMonth:o,onPreviousClick:function(){o&&a(o)},onNextClick:function(){r&&a(r)}})}function ef(e){var n,t,o=Q(),r=o.classNames,a=o.disableNavigation,i=o.styles,l=o.captionLayout,s=o.components,d=null!==(n=null==s?void 0:s.CaptionLabel)&&void 0!==n?n:V;return t=a?Y.jsx(d,{id:e.id,displayMonth:e.displayMonth}):"dropdown"===l?Y.jsx(ei,{displayMonth:e.displayMonth,id:e.id}):"dropdown-buttons"===l?Y.jsxs(Y.Fragment,{children:[Y.jsx(ei,{displayMonth:e.displayMonth,displayIndex:e.displayIndex,id:e.id}),Y.jsx(eu,{displayMonth:e.displayMonth,displayIndex:e.displayIndex,id:e.id})]}):Y.jsxs(Y.Fragment,{children:[Y.jsx(d,{id:e.id,displayMonth:e.displayMonth,displayIndex:e.displayIndex}),Y.jsx(eu,{displayMonth:e.displayMonth,id:e.id})]}),Y.jsx("div",{className:r.caption,style:i.caption,children:t})}function ep(e){var n=Q(),t=n.footer,o=n.styles,r=n.classNames.tfoot;return t?Y.jsx("tfoot",{className:r,style:o.tfoot,children:Y.jsx("tr",{children:Y.jsx("td",{colSpan:8,children:t})})}):Y.jsx(Y.Fragment,{})}function ev(){var e=Q(),n=e.classNames,t=e.styles,o=e.showWeekNumber,r=e.locale,a=e.weekStartsOn,i=e.ISOWeek,l=e.formatters.formatWeekdayName,s=e.labels.labelWeekday,d=function(e,n,t){for(var o=t?(0,x.Z)(new Date):(0,_.Z)(new Date,{locale:e,weekStartsOn:n}),r=[],a=0;a<7;a++){var i=(0,w.Z)(o,a);r.push(i)}return r}(r,a,i);return Y.jsxs("tr",{style:t.head_row,className:n.head_row,children:[o&&Y.jsx("td",{style:t.head_cell,className:n.head_cell}),d.map(function(e,o){return Y.jsx("th",{scope:"col",className:n.head_cell,style:t.head_cell,"aria-label":s(e,{locale:r}),children:l(e,{locale:r})},o)})]})}function eh(){var e,n=Q(),t=n.classNames,o=n.styles,r=n.components,a=null!==(e=null==r?void 0:r.HeadRow)&&void 0!==e?e:ev;return Y.jsx("thead",{style:o.head,className:t.head,children:Y.jsx(a,{})})}function ey(e){var n=Q(),t=n.locale,o=n.formatters.formatDay;return Y.jsx(Y.Fragment,{children:o(e.date,{locale:t})})}var em=(0,i.createContext)(void 0);function eb(e){return H(e.initialProps)?Y.jsx(ex,{initialProps:e.initialProps,children:e.children}):Y.jsx(em.Provider,{value:{selected:void 0,modifiers:{disabled:[]}},children:e.children})}function ex(e){var n=e.initialProps,t=e.children,o=n.selected,r=n.min,a=n.max,i={disabled:[]};return o&&i.disabled.push(function(e){var n=a&&o.length>a-1,t=o.some(function(n){return(0,g.Z)(n,e)});return!!(n&&!t)}),Y.jsx(em.Provider,{value:{selected:o,onDayClick:function(e,t,i){if(null===(l=n.onDayClick)||void 0===l||l.call(n,e,t,i),(!t.selected||!r||(null==o?void 0:o.length)!==r)&&(t.selected||!a||(null==o?void 0:o.length)!==a)){var l,s,d=o?R([],o,!0):[];if(t.selected){var c=d.findIndex(function(n){return(0,g.Z)(e,n)});d.splice(c,1)}else d.push(e);null===(s=n.onSelect)||void 0===s||s.call(n,d,e,t,i)}},modifiers:i},children:t})}function e_(){var e=(0,i.useContext)(em);if(!e)throw Error("useSelectMultiple must be used within a SelectMultipleProvider");return e}var ew=(0,i.createContext)(void 0);function eg(e){return K(e.initialProps)?Y.jsx(ej,{initialProps:e.initialProps,children:e.children}):Y.jsx(ew.Provider,{value:{selected:void 0,modifiers:{range_start:[],range_end:[],range_middle:[],disabled:[]}},children:e.children})}function ej(e){var n=e.initialProps,t=e.children,o=n.selected,r=o||{},a=r.from,i=r.to,l=n.min,s=n.max,d={range_start:[],range_end:[],range_middle:[],disabled:[]};if(a?(d.range_start=[a],i?(d.range_end=[i],(0,g.Z)(a,i)||(d.range_middle=[{after:a,before:i}])):d.range_end=[a]):i&&(d.range_start=[i],d.range_end=[i]),l&&(a&&!i&&d.disabled.push({after:(0,Z.Z)(a,l-1),before:(0,w.Z)(a,l-1)}),a&&i&&d.disabled.push({after:a,before:(0,w.Z)(a,l-1)}),!a&&i&&d.disabled.push({after:(0,Z.Z)(i,l-1),before:(0,w.Z)(i,l-1)})),s){if(a&&!i&&(d.disabled.push({before:(0,w.Z)(a,-s+1)}),d.disabled.push({after:(0,w.Z)(a,s-1)})),a&&i){var c=s-((0,N.Z)(i,a)+1);d.disabled.push({before:(0,Z.Z)(a,c)}),d.disabled.push({after:(0,w.Z)(i,c)})}!a&&i&&(d.disabled.push({before:(0,w.Z)(i,-s+1)}),d.disabled.push({after:(0,w.Z)(i,s-1)}))}return Y.jsx(ew.Provider,{value:{selected:o,onDayClick:function(e,t,r){null===(s=n.onDayClick)||void 0===s||s.call(n,e,t,r);var a,i,l,s,d,c=(i=(a=o||{}).from,l=a.to,i&&l?(0,g.Z)(l,e)&&(0,g.Z)(i,e)?void 0:(0,g.Z)(l,e)?{from:l,to:void 0}:(0,g.Z)(i,e)?void 0:(0,j.Z)(i,e)?{from:e,to:l}:{from:i,to:e}:l?(0,j.Z)(e,l)?{from:l,to:e}:{from:e,to:l}:i?(0,b.Z)(e,i)?{from:e,to:i}:{from:i,to:e}:{from:e,to:void 0});null===(d=n.onSelect)||void 0===d||d.call(n,c,e,t,r)},modifiers:d},children:t})}function eZ(){var e=(0,i.useContext)(ew);if(!e)throw Error("useSelectRange must be used within a SelectRangeProvider");return e}function eN(e){return Array.isArray(e)?R([],e,!0):void 0!==e?[e]:[]}(o=a||(a={})).Outside="outside",o.Disabled="disabled",o.Selected="selected",o.Hidden="hidden",o.Today="today",o.RangeStart="range_start",o.RangeEnd="range_end",o.RangeMiddle="range_middle";var ek=a.Selected,eM=a.Disabled,eD=a.Hidden,eC=a.Today,eP=a.RangeEnd,eO=a.RangeMiddle,eS=a.RangeStart,eL=a.Outside,eW=(0,i.createContext)(void 0);function eE(e){var n,t,o,r=Q(),a=e_(),i=eZ(),l=((n={})[ek]=eN(r.selected),n[eM]=eN(r.disabled),n[eD]=eN(r.hidden),n[eC]=[r.today],n[eP]=[],n[eO]=[],n[eS]=[],n[eL]=[],r.fromDate&&n[eM].push({before:r.fromDate}),r.toDate&&n[eM].push({after:r.toDate}),H(r)?n[eM]=n[eM].concat(a.modifiers[eM]):K(r)&&(n[eM]=n[eM].concat(i.modifiers[eM]),n[eS]=i.modifiers[eS],n[eO]=i.modifiers[eO],n[eP]=i.modifiers[eP]),n),s=(t=r.modifiers,o={},Object.entries(t).forEach(function(e){var n=e[0],t=e[1];o[n]=eN(t)}),o),d=T(T({},l),s);return Y.jsx(eW.Provider,{value:d,children:e.children})}function eI(){var e=(0,i.useContext)(eW);if(!e)throw Error("useModifiers must be used within a ModifiersProvider");return e}function eF(e,n,t){var o=Object.keys(n).reduce(function(t,o){return n[o].some(function(n){if("boolean"==typeof n)return n;if((0,k.Z)(n))return(0,g.Z)(e,n);if(Array.isArray(n)&&n.every(k.Z))return n.includes(e);if(n&&"object"==typeof n&&"from"in n)return o=n.from,r=n.to,o&&r?(0>(0,N.Z)(r,o)&&(o=(t=[r,o])[0],r=t[1]),(0,N.Z)(e,o)>=0&&(0,N.Z)(r,e)>=0):r?(0,g.Z)(r,e):!!o&&(0,g.Z)(o,e);if(n&&"object"==typeof n&&"dayOfWeek"in n)return n.dayOfWeek.includes(e.getDay());if(n&&"object"==typeof n&&"before"in n&&"after"in n){var t,o,r,a=(0,N.Z)(n.before,e),i=(0,N.Z)(n.after,e),l=a>0,s=i<0;return(0,j.Z)(n.before,n.after)?s&&l:l||s}return n&&"object"==typeof n&&"after"in n?(0,N.Z)(e,n.after)>0:n&&"object"==typeof n&&"before"in n?(0,N.Z)(n.before,e)>0:"function"==typeof n&&n(e)})&&t.push(o),t},[]),r={};return o.forEach(function(e){return r[e]=!0}),t&&!(0,m.Z)(e,t)&&(r.outside=!0),r}var eT=(0,i.createContext)(void 0);function eR(e){var n=ea(),t=eI(),o=(0,i.useState)(),r=o[0],a=o[1],l=(0,i.useState)(),c=l[0],u=l[1],f=function(e,n){for(var t,o,r=(0,s.Z)(e[0]),a=(0,d.Z)(e[e.length-1]),i=r;i<=a;){var l=eF(i,n);if(!(!l.disabled&&!l.hidden)){i=(0,w.Z)(i,1);continue}if(l.selected)return i;l.today&&!o&&(o=i),t||(t=i),i=(0,w.Z)(i,1)}return o||t}(n.displayMonths,t),p=(null!=r?r:c&&n.isDateDisplayed(c))?c:f,v=function(e){a(e)},h=Q(),m=function(e,o){if(r){var a=function e(n,t){var o=t.moveBy,r=t.direction,a=t.context,i=t.modifiers,l=t.retry,s=void 0===l?{count:0,lastFocused:n}:l,d=a.weekStartsOn,c=a.fromDate,u=a.toDate,f=a.locale,p=({day:w.Z,week:M.Z,month:y.Z,year:D.Z,startOfWeek:function(e){return a.ISOWeek?(0,x.Z)(e):(0,_.Z)(e,{locale:f,weekStartsOn:d})},endOfWeek:function(e){return a.ISOWeek?(0,C.Z)(e):(0,P.Z)(e,{locale:f,weekStartsOn:d})}})[o](n,"after"===r?1:-1);"before"===r&&c?p=(0,O.Z)([c,p]):"after"===r&&u&&(p=(0,S.Z)([u,p]));var v=!0;if(i){var h=eF(p,i);v=!h.disabled&&!h.hidden}return v?p:s.count>365?s.lastFocused:e(p,{moveBy:o,direction:r,context:a,modifiers:i,retry:T(T({},s),{count:s.count+1})})}(r,{moveBy:e,direction:o,context:h,modifiers:t});(0,g.Z)(r,a)||(n.goToDate(a,r),v(a))}};return Y.jsx(eT.Provider,{value:{focusedDay:r,focusTarget:p,blur:function(){u(r),a(void 0)},focus:v,focusDayAfter:function(){return m("day","after")},focusDayBefore:function(){return m("day","before")},focusWeekAfter:function(){return m("week","after")},focusWeekBefore:function(){return m("week","before")},focusMonthBefore:function(){return m("month","before")},focusMonthAfter:function(){return m("month","after")},focusYearBefore:function(){return m("year","before")},focusYearAfter:function(){return m("year","after")},focusStartOfWeek:function(){return m("startOfWeek","before")},focusEndOfWeek:function(){return m("endOfWeek","after")}},children:e.children})}function eA(){var e=(0,i.useContext)(eT);if(!e)throw Error("useFocusContext must be used within a FocusProvider");return e}var eB=(0,i.createContext)(void 0);function eY(e){return U(e.initialProps)?Y.jsx(eH,{initialProps:e.initialProps,children:e.children}):Y.jsx(eB.Provider,{value:{selected:void 0},children:e.children})}function eH(e){var n=e.initialProps,t=e.children,o={selected:n.selected,onDayClick:function(e,t,o){var r,a,i;if(null===(r=n.onDayClick)||void 0===r||r.call(n,e,t,o),t.selected&&!n.required){null===(a=n.onSelect)||void 0===a||a.call(n,void 0,e,t,o);return}null===(i=n.onSelect)||void 0===i||i.call(n,e,e,t,o)}};return Y.jsx(eB.Provider,{value:o,children:t})}function eK(){var e=(0,i.useContext)(eB);if(!e)throw Error("useSelectSingle must be used within a SelectSingleProvider");return e}function eU(e){var n,t,o,r,l,s,d,c,u,f,p,v,h,y,m,b,x,_,w,j,Z,N,k,M,D,C,P,O,S,L,W,E,I,F,R,A,B,z,G,$,q,J=(0,i.useRef)(null),V=(n=e.date,t=e.displayMonth,s=Q(),d=eA(),c=eF(n,eI(),t),u=Q(),f=eK(),p=e_(),v=eZ(),y=(h=eA()).focusDayAfter,m=h.focusDayBefore,b=h.focusWeekAfter,x=h.focusWeekBefore,_=h.blur,w=h.focus,j=h.focusMonthBefore,Z=h.focusMonthAfter,N=h.focusYearBefore,k=h.focusYearAfter,M=h.focusStartOfWeek,D=h.focusEndOfWeek,C=Q(),P=eK(),O=e_(),S=eZ(),L=U(C)?P.selected:H(C)?O.selected:K(C)?S.selected:void 0,W=!!(s.onDayClick||"default"!==s.mode),(0,i.useEffect)(function(){var e;!c.outside&&d.focusedDay&&W&&(0,g.Z)(d.focusedDay,n)&&(null===(e=J.current)||void 0===e||e.focus())},[d.focusedDay,n,J,W,c.outside]),I=(E=[s.classNames.day],Object.keys(c).forEach(function(e){var n=s.modifiersClassNames[e];if(n)E.push(n);else if(Object.values(a).includes(e)){var t=s.classNames["day_".concat(e)];t&&E.push(t)}}),E).join(" "),F=T({},s.styles.day),Object.keys(c).forEach(function(e){var n;F=T(T({},F),null===(n=s.modifiersStyles)||void 0===n?void 0:n[e])}),R=F,A=!!(c.outside&&!s.showOutsideDays||c.hidden),B=null!==(l=null===(r=s.components)||void 0===r?void 0:r.DayContent)&&void 0!==l?l:ey,z={style:R,className:I,children:Y.jsx(B,{date:n,displayMonth:t,activeModifiers:c}),role:"gridcell"},G=d.focusTarget&&(0,g.Z)(d.focusTarget,n)&&!c.outside,$=d.focusedDay&&(0,g.Z)(d.focusedDay,n),q=T(T(T({},z),((o={disabled:c.disabled,role:"gridcell"})["aria-selected"]=c.selected,o.tabIndex=$||G?0:-1,o)),{onClick:function(e){var t,o,r,a;U(u)?null===(t=f.onDayClick)||void 0===t||t.call(f,n,c,e):H(u)?null===(o=p.onDayClick)||void 0===o||o.call(p,n,c,e):K(u)?null===(r=v.onDayClick)||void 0===r||r.call(v,n,c,e):null===(a=u.onDayClick)||void 0===a||a.call(u,n,c,e)},onFocus:function(e){var t;w(n),null===(t=u.onDayFocus)||void 0===t||t.call(u,n,c,e)},onBlur:function(e){var t;_(),null===(t=u.onDayBlur)||void 0===t||t.call(u,n,c,e)},onKeyDown:function(e){var t;switch(e.key){case"ArrowLeft":e.preventDefault(),e.stopPropagation(),"rtl"===u.dir?y():m();break;case"ArrowRight":e.preventDefault(),e.stopPropagation(),"rtl"===u.dir?m():y();break;case"ArrowDown":e.preventDefault(),e.stopPropagation(),b();break;case"ArrowUp":e.preventDefault(),e.stopPropagation(),x();break;case"PageUp":e.preventDefault(),e.stopPropagation(),e.shiftKey?N():j();break;case"PageDown":e.preventDefault(),e.stopPropagation(),e.shiftKey?k():Z();break;case"Home":e.preventDefault(),e.stopPropagation(),M();break;case"End":e.preventDefault(),e.stopPropagation(),D()}null===(t=u.onDayKeyDown)||void 0===t||t.call(u,n,c,e)},onKeyUp:function(e){var t;null===(t=u.onDayKeyUp)||void 0===t||t.call(u,n,c,e)},onMouseEnter:function(e){var t;null===(t=u.onDayMouseEnter)||void 0===t||t.call(u,n,c,e)},onMouseLeave:function(e){var t;null===(t=u.onDayMouseLeave)||void 0===t||t.call(u,n,c,e)},onPointerEnter:function(e){var t;null===(t=u.onDayPointerEnter)||void 0===t||t.call(u,n,c,e)},onPointerLeave:function(e){var t;null===(t=u.onDayPointerLeave)||void 0===t||t.call(u,n,c,e)},onTouchCancel:function(e){var t;null===(t=u.onDayTouchCancel)||void 0===t||t.call(u,n,c,e)},onTouchEnd:function(e){var t;null===(t=u.onDayTouchEnd)||void 0===t||t.call(u,n,c,e)},onTouchMove:function(e){var t;null===(t=u.onDayTouchMove)||void 0===t||t.call(u,n,c,e)},onTouchStart:function(e){var t;null===(t=u.onDayTouchStart)||void 0===t||t.call(u,n,c,e)}}),{isButton:W,isHidden:A,activeModifiers:c,selectedDays:L,buttonProps:q,divProps:z});return V.isHidden?Y.jsx("div",{role:"gridcell"}):V.isButton?Y.jsx(ed,T({name:"day",ref:J},V.buttonProps)):Y.jsx("div",T({},V.divProps))}function ez(e){var n=e.number,t=e.dates,o=Q(),r=o.onWeekNumberClick,a=o.styles,i=o.classNames,l=o.locale,s=o.labels.labelWeekNumber,d=(0,o.formatters.formatWeekNumber)(Number(n),{locale:l});if(!r)return Y.jsx("span",{className:i.weeknumber,style:a.weeknumber,children:d});var c=s(Number(n),{locale:l});return Y.jsx(ed,{name:"week-number","aria-label":c,className:i.weeknumber,style:a.weeknumber,onClick:function(e){r(n,t,e)},children:d})}function eG(e){var n,t,o,r=Q(),a=r.styles,i=r.classNames,l=r.showWeekNumber,s=r.components,d=null!==(n=null==s?void 0:s.Day)&&void 0!==n?n:eU,c=null!==(t=null==s?void 0:s.WeekNumber)&&void 0!==t?t:ez;return l&&(o=Y.jsx("td",{className:i.cell,style:a.cell,children:Y.jsx(c,{number:e.weekNumber,dates:e.dates})})),Y.jsxs("tr",{className:i.row,style:a.row,children:[o,e.dates.map(function(n){return Y.jsx("td",{className:i.cell,style:a.cell,role:"presentation",children:Y.jsx(d,{displayMonth:e.displayMonth,date:n})},(0,L.Z)(n))})]})}function e$(e,n,t){for(var o=(null==t?void 0:t.ISOWeek)?(0,C.Z)(n):(0,P.Z)(n,t),r=(null==t?void 0:t.ISOWeek)?(0,x.Z)(e):(0,_.Z)(e,t),a=(0,N.Z)(o,r),i=[],l=0;l<=a;l++)i.push((0,w.Z)(r,l));return i.reduce(function(e,n){var o=(null==t?void 0:t.ISOWeek)?(0,W.Z)(n):(0,E.Z)(n,t),r=e.find(function(e){return e.weekNumber===o});return r?r.dates.push(n):e.push({weekNumber:o,dates:[n]}),e},[])}function eq(e){var n,t,o,r=Q(),a=r.locale,i=r.classNames,l=r.styles,c=r.hideHead,u=r.fixedWeeks,f=r.components,p=r.weekStartsOn,v=r.firstWeekContainsDate,h=r.ISOWeek,y=function(e,n){var t=e$((0,s.Z)(e),(0,d.Z)(e),n);if(null==n?void 0:n.useFixedWeeks){var o=(0,I.Z)(e,n);if(o<6){var r=t[t.length-1],a=r.dates[r.dates.length-1],i=(0,M.Z)(a,6-o),l=e$((0,M.Z)(a,1),i,n);t.push.apply(t,l)}}return t}(e.displayMonth,{useFixedWeeks:!!u,ISOWeek:h,locale:a,weekStartsOn:p,firstWeekContainsDate:v}),m=null!==(n=null==f?void 0:f.Head)&&void 0!==n?n:eh,b=null!==(t=null==f?void 0:f.Row)&&void 0!==t?t:eG,x=null!==(o=null==f?void 0:f.Footer)&&void 0!==o?o:ep;return Y.jsxs("table",{id:e.id,className:i.table,style:l.table,role:"grid","aria-labelledby":e["aria-labelledby"],children:[!c&&Y.jsx(m,{}),Y.jsx("tbody",{className:i.tbody,style:l.tbody,children:y.map(function(n){return Y.jsx(b,{displayMonth:e.displayMonth,dates:n.dates,weekNumber:n.weekNumber},n.weekNumber)})}),Y.jsx(x,{displayMonth:e.displayMonth})]})}var eJ=window.document&&window.document.createElement?i.useLayoutEffect:i.useEffect,eQ=!1,eV=0;function eX(){return"react-day-picker-".concat(++eV)}function e0(e){var n,t,o,r,a,l,s,d,c=Q(),u=c.dir,f=c.classNames,p=c.styles,v=c.components,h=ea().displayMonths,y=(o=null!=(n=c.id?"".concat(c.id,"-").concat(e.displayIndex):void 0)?n:eQ?eX():null,a=(r=(0,i.useState)(o))[0],l=r[1],eJ(function(){null===a&&l(eX())},[]),(0,i.useEffect)(function(){!1===eQ&&(eQ=!0)},[]),null!==(t=null!=n?n:a)&&void 0!==t?t:void 0),m=c.id?"".concat(c.id,"-grid-").concat(e.displayIndex):void 0,b=[f.month],x=p.month,_=0===e.displayIndex,w=e.displayIndex===h.length-1,g=!_&&!w;"rtl"===u&&(w=(s=[_,w])[0],_=s[1]),_&&(b.push(f.caption_start),x=T(T({},x),p.caption_start)),w&&(b.push(f.caption_end),x=T(T({},x),p.caption_end)),g&&(b.push(f.caption_between),x=T(T({},x),p.caption_between));var j=null!==(d=null==v?void 0:v.Caption)&&void 0!==d?d:ef;return Y.jsxs("div",{className:b.join(" "),style:x,children:[Y.jsx(j,{id:y,displayMonth:e.displayMonth,displayIndex:e.displayIndex}),Y.jsx(eq,{id:m,"aria-labelledby":y,displayMonth:e.displayMonth})]},e.displayIndex)}function e1(e){var n=Q(),t=n.classNames,o=n.styles;return Y.jsx("div",{className:t.months,style:o.months,children:e.children})}function e4(e){var n,t,o=e.initialProps,r=Q(),a=eA(),l=ea(),s=(0,i.useState)(!1),d=s[0],c=s[1];(0,i.useEffect)(function(){r.initialFocus&&a.focusTarget&&(d||(a.focus(a.focusTarget),c(!0)))},[r.initialFocus,d,a.focus,a.focusTarget,a]);var u=[r.classNames.root,r.className];r.numberOfMonths>1&&u.push(r.classNames.multiple_months),r.showWeekNumber&&u.push(r.classNames.with_weeknumber);var f=T(T({},r.styles.root),r.style),p=Object.keys(o).filter(function(e){return e.startsWith("data-")}).reduce(function(e,n){var t;return T(T({},e),((t={})[n]=o[n],t))},{}),v=null!==(t=null===(n=o.components)||void 0===n?void 0:n.Months)&&void 0!==t?t:e1;return Y.jsx("div",T({className:u.join(" "),style:f,dir:r.dir,id:r.id,nonce:o.nonce,title:o.title,lang:o.lang},p,{children:Y.jsx(v,{children:l.displayMonths.map(function(e,n){return Y.jsx(e0,{displayIndex:n,displayMonth:e},n)})})}))}function e5(e){var n=e.children,t=function(e,n){var t={};for(var o in e)Object.prototype.hasOwnProperty.call(e,o)&&0>n.indexOf(o)&&(t[o]=e[o]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,o=Object.getOwnPropertySymbols(e);rn.indexOf(o[r])&&Object.prototype.propertyIsEnumerable.call(e,o[r])&&(t[o[r]]=e[o[r]]);return t}(e,["children"]);return Y.jsx(J,{initialProps:t,children:Y.jsx(er,{children:Y.jsx(eY,{initialProps:t,children:Y.jsx(eb,{initialProps:t,children:Y.jsx(eg,{initialProps:t,children:Y.jsx(eE,{children:Y.jsx(eR,{children:n})})})})})})})}function e8(e){return Y.jsx(e5,T({},e,{children:Y.jsx(e4,{initialProps:e})}))}}}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/386-d811195b597a2122.js b/litellm/proxy/_experimental/out/_next/static/chunks/386-d811195b597a2122.js deleted file mode 100644 index c589a3c73..000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/386-d811195b597a2122.js +++ /dev/null @@ -1,32 +0,0 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[386],{12215:function(e,t,n){n.d(t,{iN:function(){return h},R_:function(){return d},EV:function(){return g},ez:function(){return p}});var r=n(41785),o=n(76991),a=[{index:7,opacity:.15},{index:6,opacity:.25},{index:5,opacity:.3},{index:5,opacity:.45},{index:5,opacity:.65},{index:5,opacity:.85},{index:4,opacity:.9},{index:3,opacity:.95},{index:2,opacity:.97},{index:1,opacity:.98}];function i(e){var t=e.r,n=e.g,o=e.b,a=(0,r.py)(t,n,o);return{h:360*a.h,s:a.s,v:a.v}}function l(e){var t=e.r,n=e.g,o=e.b;return"#".concat((0,r.vq)(t,n,o,!1))}function s(e,t,n){var r;return(r=Math.round(e.h)>=60&&240>=Math.round(e.h)?n?Math.round(e.h)-2*t:Math.round(e.h)+2*t:n?Math.round(e.h)+2*t:Math.round(e.h)-2*t)<0?r+=360:r>=360&&(r-=360),r}function c(e,t,n){var r;return 0===e.h&&0===e.s?e.s:((r=n?e.s-.16*t:4===t?e.s+.16:e.s+.05*t)>1&&(r=1),n&&5===t&&r>.1&&(r=.1),r<.06&&(r=.06),Number(r.toFixed(2)))}function u(e,t,n){var r;return(r=n?e.v+.05*t:e.v-.15*t)>1&&(r=1),Number(r.toFixed(2))}function d(e){for(var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=[],r=(0,o.uA)(e),d=5;d>0;d-=1){var p=i(r),f=l((0,o.uA)({h:s(p,d,!0),s:c(p,d,!0),v:u(p,d,!0)}));n.push(f)}n.push(l(r));for(var m=1;m<=4;m+=1){var g=i(r),h=l((0,o.uA)({h:s(g,m),s:c(g,m),v:u(g,m)}));n.push(h)}return"dark"===t.theme?a.map(function(e){var r,a,i,s=e.index,c=e.opacity;return l((r=(0,o.uA)(t.backgroundColor||"#141414"),a=(0,o.uA)(n[s]),i=100*c/100,{r:(a.r-r.r)*i+r.r,g:(a.g-r.g)*i+r.g,b:(a.b-r.b)*i+r.b}))}):n}var p={red:"#F5222D",volcano:"#FA541C",orange:"#FA8C16",gold:"#FAAD14",yellow:"#FADB14",lime:"#A0D911",green:"#52C41A",cyan:"#13C2C2",blue:"#1677FF",geekblue:"#2F54EB",purple:"#722ED1",magenta:"#EB2F96",grey:"#666666"},f={},m={};Object.keys(p).forEach(function(e){f[e]=d(p[e]),f[e].primary=f[e][5],m[e]=d(p[e],{theme:"dark",backgroundColor:"#141414"}),m[e].primary=m[e][5]}),f.red,f.volcano;var g=f.gold;f.orange,f.yellow,f.lime,f.green,f.cyan;var h=f.blue;f.geekblue,f.purple,f.magenta,f.grey,f.grey},8985:function(e,t,n){n.d(t,{E4:function(){return ej},jG:function(){return T},ks:function(){return Z},bf:function(){return F},CI:function(){return eD},fp:function(){return X},xy:function(){return eL}});var r,o,a=n(50833),i=n(80406),l=n(63787),s=n(5239),c=function(e){for(var t,n=0,r=0,o=e.length;o>=4;++r,o-=4)t=(65535&(t=255&e.charCodeAt(r)|(255&e.charCodeAt(++r))<<8|(255&e.charCodeAt(++r))<<16|(255&e.charCodeAt(++r))<<24))*1540483477+((t>>>16)*59797<<16),t^=t>>>24,n=(65535&t)*1540483477+((t>>>16)*59797<<16)^(65535&n)*1540483477+((n>>>16)*59797<<16);switch(o){case 3:n^=(255&e.charCodeAt(r+2))<<16;case 2:n^=(255&e.charCodeAt(r+1))<<8;case 1:n^=255&e.charCodeAt(r),n=(65535&n)*1540483477+((n>>>16)*59797<<16)}return n^=n>>>13,(((n=(65535&n)*1540483477+((n>>>16)*59797<<16))^n>>>15)>>>0).toString(36)},u=n(24050),d=n(64090),p=n.t(d,2);n(61475),n(92536);var f=n(47365),m=n(65127);function g(e){return e.join("%")}var h=function(){function e(t){(0,f.Z)(this,e),(0,a.Z)(this,"instanceId",void 0),(0,a.Z)(this,"cache",new Map),this.instanceId=t}return(0,m.Z)(e,[{key:"get",value:function(e){return this.opGet(g(e))}},{key:"opGet",value:function(e){return this.cache.get(e)||null}},{key:"update",value:function(e,t){return this.opUpdate(g(e),t)}},{key:"opUpdate",value:function(e,t){var n=t(this.cache.get(e));null===n?this.cache.delete(e):this.cache.set(e,n)}}]),e}(),b="data-token-hash",v="data-css-hash",y="__cssinjs_instance__",E=d.createContext({hashPriority:"low",cache:function(){var e=Math.random().toString(12).slice(2);if("undefined"!=typeof document&&document.head&&document.body){var t=document.body.querySelectorAll("style[".concat(v,"]"))||[],n=document.head.firstChild;Array.from(t).forEach(function(t){t[y]=t[y]||e,t[y]===e&&document.head.insertBefore(t,n)});var r={};Array.from(document.querySelectorAll("style[".concat(v,"]"))).forEach(function(t){var n,o=t.getAttribute(v);r[o]?t[y]===e&&(null===(n=t.parentNode)||void 0===n||n.removeChild(t)):r[o]=!0})}return new h(e)}(),defaultCache:!0}),S=n(6976),w=n(22127),x=function(){function e(){(0,f.Z)(this,e),(0,a.Z)(this,"cache",void 0),(0,a.Z)(this,"keys",void 0),(0,a.Z)(this,"cacheCallTimes",void 0),this.cache=new Map,this.keys=[],this.cacheCallTimes=0}return(0,m.Z)(e,[{key:"size",value:function(){return this.keys.length}},{key:"internalGet",value:function(e){var t,n,r=arguments.length>1&&void 0!==arguments[1]&&arguments[1],o={map:this.cache};return e.forEach(function(e){if(o){var t;o=null===(t=o)||void 0===t||null===(t=t.map)||void 0===t?void 0:t.get(e)}else o=void 0}),null!==(t=o)&&void 0!==t&&t.value&&r&&(o.value[1]=this.cacheCallTimes++),null===(n=o)||void 0===n?void 0:n.value}},{key:"get",value:function(e){var t;return null===(t=this.internalGet(e,!0))||void 0===t?void 0:t[0]}},{key:"has",value:function(e){return!!this.internalGet(e)}},{key:"set",value:function(t,n){var r=this;if(!this.has(t)){if(this.size()+1>e.MAX_CACHE_SIZE+e.MAX_CACHE_OFFSET){var o=this.keys.reduce(function(e,t){var n=(0,i.Z)(e,2)[1];return r.internalGet(t)[1]0,"[Ant Design CSS-in-JS] Theme should have at least one derivative function."),k+=1}return(0,m.Z)(e,[{key:"getDerivativeToken",value:function(e){return this.derivatives.reduce(function(t,n){return n(e,t)},void 0)}}]),e}(),A=new x;function T(e){var t=Array.isArray(e)?e:[e];return A.has(t)||A.set(t,new C(t)),A.get(t)}var I=new WeakMap,N={},R=new WeakMap;function _(e){var t=R.get(e)||"";return t||(Object.keys(e).forEach(function(n){var r=e[n];t+=n,r instanceof C?t+=r.id:r&&"object"===(0,S.Z)(r)?t+=_(r):t+=r}),R.set(e,t)),t}function P(e,t){return c("".concat(t,"_").concat(_(e)))}var L="random-".concat(Date.now(),"-").concat(Math.random()).replace(/\./g,""),M="_bAmBoO_",D=void 0,j=(0,w.Z)();function F(e){return"number"==typeof e?"".concat(e,"px"):e}function B(e,t,n){var r,o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},i=arguments.length>4&&void 0!==arguments[4]&&arguments[4];if(i)return e;var l=(0,s.Z)((0,s.Z)({},o),{},(r={},(0,a.Z)(r,b,t),(0,a.Z)(r,v,n),r)),c=Object.keys(l).map(function(e){var t=l[e];return t?"".concat(e,'="').concat(t,'"'):null}).filter(function(e){return e}).join(" ");return"")}var Z=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"";return"--".concat(t?"".concat(t,"-"):"").concat(e).replace(/([a-z0-9])([A-Z])/g,"$1-$2").replace(/([A-Z]+)([A-Z][a-z0-9]+)/g,"$1-$2").replace(/([a-z])([A-Z0-9])/g,"$1-$2").toLowerCase()},U=function(e,t,n){var r,o={},a={};return Object.entries(e).forEach(function(e){var t=(0,i.Z)(e,2),r=t[0],l=t[1];if(null!=n&&null!==(s=n.preserve)&&void 0!==s&&s[r])a[r]=l;else if(("string"==typeof l||"number"==typeof l)&&!(null!=n&&null!==(c=n.ignore)&&void 0!==c&&c[r])){var s,c,u,d=Z(r,null==n?void 0:n.prefix);o[d]="number"!=typeof l||null!=n&&null!==(u=n.unitless)&&void 0!==u&&u[r]?String(l):"".concat(l,"px"),a[r]="var(".concat(d,")")}}),[a,(r={scope:null==n?void 0:n.scope},Object.keys(o).length?".".concat(t).concat(null!=r&&r.scope?".".concat(r.scope):"","{").concat(Object.entries(o).map(function(e){var t=(0,i.Z)(e,2),n=t[0],r=t[1];return"".concat(n,":").concat(r,";")}).join(""),"}"):"")]},z=n(24800),H=(0,s.Z)({},p).useInsertionEffect,G=H?function(e,t,n){return H(function(){return e(),t()},n)}:function(e,t,n){d.useMemo(e,n),(0,z.Z)(function(){return t(!0)},n)},$=void 0!==(0,s.Z)({},p).useInsertionEffect?function(e){var t=[],n=!1;return d.useEffect(function(){return n=!1,function(){n=!0,t.length&&t.forEach(function(e){return e()})}},e),function(e){n||t.push(e)}}:function(){return function(e){e()}};function W(e,t,n,r,o){var a=d.useContext(E).cache,s=g([e].concat((0,l.Z)(t))),c=$([s]),u=function(e){a.opUpdate(s,function(t){var r=(0,i.Z)(t||[void 0,void 0],2),o=r[0],a=[void 0===o?0:o,r[1]||n()];return e?e(a):a})};d.useMemo(function(){u()},[s]);var p=a.opGet(s)[1];return G(function(){null==o||o(p)},function(e){return u(function(t){var n=(0,i.Z)(t,2),r=n[0],a=n[1];return e&&0===r&&(null==o||o(p)),[r+1,a]}),function(){a.opUpdate(s,function(t){var n=(0,i.Z)(t||[],2),o=n[0],l=void 0===o?0:o,u=n[1];return 0==l-1?(c(function(){(e||!a.opGet(s))&&(null==r||r(u,!1))}),null):[l-1,u]})}},[s]),p}var V={},q=new Map,Y=function(e,t,n,r){var o=n.getDerivativeToken(e),a=(0,s.Z)((0,s.Z)({},o),t);return r&&(a=r(a)),a},K="token";function X(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},r=(0,d.useContext)(E),o=r.cache.instanceId,a=r.container,p=n.salt,f=void 0===p?"":p,m=n.override,g=void 0===m?V:m,h=n.formatToken,S=n.getComputedToken,w=n.cssVar,x=function(e,t){for(var n=I,r=0;r=(q.get(e)||0)}),n.length-r.length>0&&r.forEach(function(e){"undefined"!=typeof document&&document.querySelectorAll("style[".concat(b,'="').concat(e,'"]')).forEach(function(e){if(e[y]===o){var t;null===(t=e.parentNode)||void 0===t||t.removeChild(e)}}),q.delete(e)})},function(e){var t=(0,i.Z)(e,4),n=t[0],r=t[3];if(w&&r){var l=(0,u.hq)(r,c("css-variables-".concat(n._themeKey)),{mark:v,prepend:"queue",attachTo:a,priority:-999});l[y]=o,l.setAttribute(b,n._themeKey)}})}var Q=n(14749),J={animationIterationCount:1,borderImageOutset:1,borderImageSlice:1,borderImageWidth:1,boxFlex:1,boxFlexGroup:1,boxOrdinalGroup:1,columnCount:1,columns:1,flex:1,flexGrow:1,flexPositive:1,flexShrink:1,flexNegative:1,flexOrder:1,gridRow:1,gridRowEnd:1,gridRowSpan:1,gridRowStart:1,gridColumn:1,gridColumnEnd:1,gridColumnSpan:1,gridColumnStart:1,msGridRow:1,msGridRowSpan:1,msGridColumn:1,msGridColumnSpan:1,fontWeight:1,lineHeight:1,opacity:1,order:1,orphans:1,tabSize:1,widows:1,zIndex:1,zoom:1,WebkitLineClamp:1,fillOpacity:1,floodOpacity:1,stopOpacity:1,strokeDasharray:1,strokeDashoffset:1,strokeMiterlimit:1,strokeOpacity:1,strokeWidth:1},ee="comm",et="rule",en="decl",er=Math.abs,eo=String.fromCharCode;function ea(e,t,n){return e.replace(t,n)}function ei(e,t){return 0|e.charCodeAt(t)}function el(e,t,n){return e.slice(t,n)}function es(e){return e.length}function ec(e,t){return t.push(e),e}function eu(e,t){for(var n="",r=0;r0?f[v]+" "+y:ea(y,/&\f/g,f[v])).trim())&&(s[b++]=E);return ev(e,t,n,0===o?et:l,s,c,u,d)}function eO(e,t,n,r,o){return ev(e,t,n,en,el(e,0,r),el(e,r+1,-1),r,o)}var ek="data-ant-cssinjs-cache-path",eC="_FILE_STYLE__",eA=!0,eT="_multi_value_";function eI(e){var t,n,r;return eu((r=function e(t,n,r,o,a,i,l,s,c){for(var u,d,p,f=0,m=0,g=l,h=0,b=0,v=0,y=1,E=1,S=1,w=0,x="",O=a,k=i,C=o,A=x;E;)switch(v=w,w=ey()){case 40:if(108!=v&&58==ei(A,g-1)){-1!=(d=A+=ea(ew(w),"&","&\f"),p=er(f?s[f-1]:0),d.indexOf("&\f",p))&&(S=-1);break}case 34:case 39:case 91:A+=ew(w);break;case 9:case 10:case 13:case 32:A+=function(e){for(;eh=eE();)if(eh<33)ey();else break;return eS(e)>2||eS(eh)>3?"":" "}(v);break;case 92:A+=function(e,t){for(var n;--t&&ey()&&!(eh<48)&&!(eh>102)&&(!(eh>57)||!(eh<65))&&(!(eh>70)||!(eh<97)););return n=eg+(t<6&&32==eE()&&32==ey()),el(eb,e,n)}(eg-1,7);continue;case 47:switch(eE()){case 42:case 47:ec(ev(u=function(e,t){for(;ey();)if(e+eh===57)break;else if(e+eh===84&&47===eE())break;return"/*"+el(eb,t,eg-1)+"*"+eo(47===e?e:ey())}(ey(),eg),n,r,ee,eo(eh),el(u,2,-2),0,c),c);break;default:A+="/"}break;case 123*y:s[f++]=es(A)*S;case 125*y:case 59:case 0:switch(w){case 0:case 125:E=0;case 59+m:-1==S&&(A=ea(A,/\f/g,"")),b>0&&es(A)-g&&ec(b>32?eO(A+";",o,r,g-1,c):eO(ea(A," ","")+";",o,r,g-2,c),c);break;case 59:A+=";";default:if(ec(C=ex(A,n,r,f,m,a,s,x,O=[],k=[],g,i),i),123===w){if(0===m)e(A,n,C,C,O,i,g,s,k);else switch(99===h&&110===ei(A,3)?100:h){case 100:case 108:case 109:case 115:e(t,C,C,o&&ec(ex(t,C,C,0,0,a,s,x,a,O=[],g,k),k),a,k,g,s,o?O:k);break;default:e(A,C,C,C,[""],k,0,s,k)}}}f=m=b=0,y=S=1,x=A="",g=l;break;case 58:g=1+es(A),b=v;default:if(y<1){if(123==w)--y;else if(125==w&&0==y++&&125==(eh=eg>0?ei(eb,--eg):0,ef--,10===eh&&(ef=1,ep--),eh))continue}switch(A+=eo(w),w*y){case 38:S=m>0?1:(A+="\f",-1);break;case 44:s[f++]=(es(A)-1)*S,S=1;break;case 64:45===eE()&&(A+=ew(ey())),h=eE(),m=g=es(x=A+=function(e){for(;!eS(eE());)ey();return el(eb,e,eg)}(eg)),w++;break;case 45:45===v&&2==es(A)&&(y=0)}}return i}("",null,null,null,[""],(n=t=e,ep=ef=1,em=es(eb=n),eg=0,t=[]),0,[0],t),eb="",r),ed).replace(/\{%%%\:[^;];}/g,";")}var eN=function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{root:!0,parentSelectors:[]},o=r.root,a=r.injectHash,c=r.parentSelectors,d=n.hashId,p=n.layer,f=(n.path,n.hashPriority),m=n.transformers,g=void 0===m?[]:m;n.linters;var h="",b={};function v(t){var r=t.getName(d);if(!b[r]){var o=e(t.style,n,{root:!1,parentSelectors:c}),a=(0,i.Z)(o,1)[0];b[r]="@keyframes ".concat(t.getName(d)).concat(a)}}if((function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[];return t.forEach(function(t){Array.isArray(t)?e(t,n):t&&n.push(t)}),n})(Array.isArray(t)?t:[t]).forEach(function(t){var r="string"!=typeof t||o?t:{};if("string"==typeof r)h+="".concat(r,"\n");else if(r._keyframe)v(r);else{var u=g.reduce(function(e,t){var n;return(null==t||null===(n=t.visit)||void 0===n?void 0:n.call(t,e))||e},r);Object.keys(u).forEach(function(t){var r=u[t];if("object"!==(0,S.Z)(r)||!r||"animationName"===t&&r._keyframe||"object"===(0,S.Z)(r)&&r&&("_skip_check_"in r||eT in r)){function p(e,t){var n=e.replace(/[A-Z]/g,function(e){return"-".concat(e.toLowerCase())}),r=t;J[e]||"number"!=typeof r||0===r||(r="".concat(r,"px")),"animationName"===e&&null!=t&&t._keyframe&&(v(t),r=t.getName(d)),h+="".concat(n,":").concat(r,";")}var m,g=null!==(m=null==r?void 0:r.value)&&void 0!==m?m:r;"object"===(0,S.Z)(r)&&null!=r&&r[eT]&&Array.isArray(g)?g.forEach(function(e){p(t,e)}):p(t,g)}else{var y=!1,E=t.trim(),w=!1;(o||a)&&d?E.startsWith("@")?y=!0:E=function(e,t,n){if(!t)return e;var r=".".concat(t),o="low"===n?":where(".concat(r,")"):r;return e.split(",").map(function(e){var t,n=e.trim().split(/\s+/),r=n[0]||"",a=(null===(t=r.match(/^\w+/))||void 0===t?void 0:t[0])||"";return[r="".concat(a).concat(o).concat(r.slice(a.length))].concat((0,l.Z)(n.slice(1))).join(" ")}).join(",")}(t,d,f):o&&!d&&("&"===E||""===E)&&(E="",w=!0);var x=e(r,n,{root:w,injectHash:y,parentSelectors:[].concat((0,l.Z)(c),[E])}),O=(0,i.Z)(x,2),k=O[0],C=O[1];b=(0,s.Z)((0,s.Z)({},b),C),h+="".concat(E).concat(k)}})}}),o){if(p&&(void 0===D&&(D=function(e,t,n){if((0,w.Z)()){(0,u.hq)(e,L);var r,o,a=document.createElement("div");a.style.position="fixed",a.style.left="0",a.style.top="0",null==t||t(a),document.body.appendChild(a);var i=n?n(a):null===(r=getComputedStyle(a).content)||void 0===r?void 0:r.includes(M);return null===(o=a.parentNode)||void 0===o||o.removeChild(a),(0,u.jL)(L),i}return!1}("@layer ".concat(L," { .").concat(L,' { content: "').concat(M,'"!important; } }'),function(e){e.className=L})),D)){var y=p.split(","),E=y[y.length-1].trim();h="@layer ".concat(E," {").concat(h,"}"),y.length>1&&(h="@layer ".concat(p,"{%%%:%}").concat(h))}}else h="{".concat(h,"}");return[h,b]};function eR(e,t){return c("".concat(e.join("%")).concat(t))}function e_(){return null}var eP="style";function eL(e,t){var n=e.token,o=e.path,s=e.hashId,c=e.layer,p=e.nonce,f=e.clientOnly,m=e.order,g=void 0===m?0:m,h=d.useContext(E),S=h.autoClear,x=(h.mock,h.defaultCache),O=h.hashPriority,k=h.container,C=h.ssrInline,A=h.transformers,T=h.linters,I=h.cache,N=n._tokenKey,R=[N].concat((0,l.Z)(o)),_=W(eP,R,function(){var e=R.join("|");if(!function(){if(!r&&(r={},(0,w.Z)())){var e,t=document.createElement("div");t.className=ek,t.style.position="fixed",t.style.visibility="hidden",t.style.top="-9999px",document.body.appendChild(t);var n=getComputedStyle(t).content||"";(n=n.replace(/^"/,"").replace(/"$/,"")).split(";").forEach(function(e){var t=e.split(":"),n=(0,i.Z)(t,2),o=n[0],a=n[1];r[o]=a});var o=document.querySelector("style[".concat(ek,"]"));o&&(eA=!1,null===(e=o.parentNode)||void 0===e||e.removeChild(o)),document.body.removeChild(t)}}(),r[e]){var n=function(e){var t=r[e],n=null;if(t&&(0,w.Z)()){if(eA)n=eC;else{var o=document.querySelector("style[".concat(v,'="').concat(r[e],'"]'));o?n=o.innerHTML:delete r[e]}}return[n,t]}(e),a=(0,i.Z)(n,2),l=a[0],u=a[1];if(l)return[l,N,u,{},f,g]}var d=eN(t(),{hashId:s,hashPriority:O,layer:c,path:o.join("-"),transformers:A,linters:T}),p=(0,i.Z)(d,2),m=p[0],h=p[1],b=eI(m),y=eR(R,b);return[b,N,y,h,f,g]},function(e,t){var n=(0,i.Z)(e,3)[2];(t||S)&&j&&(0,u.jL)(n,{mark:v})},function(e){var t=(0,i.Z)(e,4),n=t[0],r=(t[1],t[2]),o=t[3];if(j&&n!==eC){var a={mark:v,prepend:"queue",attachTo:k,priority:g},l="function"==typeof p?p():p;l&&(a.csp={nonce:l});var s=(0,u.hq)(n,r,a);s[y]=I.instanceId,s.setAttribute(b,N),Object.keys(o).forEach(function(e){(0,u.hq)(eI(o[e]),"_effect-".concat(e),a)})}}),P=(0,i.Z)(_,3),L=P[0],M=P[1],D=P[2];return function(e){var t,n;return t=C&&!j&&x?d.createElement("style",(0,Q.Z)({},(n={},(0,a.Z)(n,b,M),(0,a.Z)(n,v,D),n),{dangerouslySetInnerHTML:{__html:L}})):d.createElement(e_,null),d.createElement(d.Fragment,null,t,e)}}var eM="cssVar",eD=function(e,t){var n=e.key,r=e.prefix,o=e.unitless,a=e.ignore,s=e.token,c=e.scope,p=void 0===c?"":c,f=(0,d.useContext)(E),m=f.cache.instanceId,g=f.container,h=s._tokenKey,S=[].concat((0,l.Z)(e.path),[n,p,h]);return W(eM,S,function(){var e=U(t(),n,{prefix:r,unitless:o,ignore:a,scope:p}),l=(0,i.Z)(e,2),s=l[0],c=l[1],u=eR(S,c);return[s,c,u,n]},function(e){var t=(0,i.Z)(e,3)[2];j&&(0,u.jL)(t,{mark:v})},function(e){var t=(0,i.Z)(e,3),r=t[1],o=t[2];if(r){var a=(0,u.hq)(r,o,{mark:v,prepend:"queue",attachTo:g,priority:-999});a[y]=m,a.setAttribute(b,n)}})};o={},(0,a.Z)(o,eP,function(e,t,n){var r=(0,i.Z)(e,6),o=r[0],a=r[1],l=r[2],s=r[3],c=r[4],u=r[5],d=(n||{}).plain;if(c)return null;var p=o,f={"data-rc-order":"prependQueue","data-rc-priority":"".concat(u)};return p=B(o,a,l,f,d),s&&Object.keys(s).forEach(function(e){if(!t[e]){t[e]=!0;var n=eI(s[e]);p+=B(n,a,"_effect-".concat(e),f,d)}}),[u,l,p]}),(0,a.Z)(o,K,function(e,t,n){var r=(0,i.Z)(e,5),o=r[2],a=r[3],l=r[4],s=(n||{}).plain;if(!a)return null;var c=o._tokenKey,u=B(a,l,c,{"data-rc-order":"prependQueue","data-rc-priority":"".concat(-999)},s);return[-999,c,u]}),(0,a.Z)(o,eM,function(e,t,n){var r=(0,i.Z)(e,4),o=r[1],a=r[2],l=r[3],s=(n||{}).plain;if(!o)return null;var c=B(o,l,a,{"data-rc-order":"prependQueue","data-rc-priority":"".concat(-999)},s);return[-999,a,c]});var ej=function(){function e(t,n){(0,f.Z)(this,e),(0,a.Z)(this,"name",void 0),(0,a.Z)(this,"style",void 0),(0,a.Z)(this,"_keyframe",!0),this.name=t,this.style=n}return(0,m.Z)(e,[{key:"getName",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"";return e?"".concat(e,"-").concat(this.name):this.name}}]),e}();function eF(e){return e.notSplit=!0,e}eF(["borderTop","borderBottom"]),eF(["borderTop"]),eF(["borderBottom"]),eF(["borderLeft","borderRight"]),eF(["borderLeft"]),eF(["borderRight"])},60688:function(e,t,n){n.d(t,{Z:function(){return T}});var r=n(14749),o=n(80406),a=n(50833),i=n(6787),l=n(64090),s=n(16480),c=n.n(s),u=n(12215),d=n(67689),p=n(5239),f=n(6976),m=n(24050),g=n(74687),h=n(53850);function b(e){return"object"===(0,f.Z)(e)&&"string"==typeof e.name&&"string"==typeof e.theme&&("object"===(0,f.Z)(e.icon)||"function"==typeof e.icon)}function v(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return Object.keys(e).reduce(function(t,n){var r=e[n];return"class"===n?(t.className=r,delete t.class):(delete t[n],t[n.replace(/-(.)/g,function(e,t){return t.toUpperCase()})]=r),t},{})}function y(e){return(0,u.R_)(e)[0]}function E(e){return e?Array.isArray(e)?e:[e]:[]}var S=function(e){var t=(0,l.useContext)(d.Z),n=t.csp,r=t.prefixCls,o="\n.anticon {\n display: inline-block;\n color: inherit;\n font-style: normal;\n line-height: 0;\n text-align: center;\n text-transform: none;\n vertical-align: -0.125em;\n text-rendering: optimizeLegibility;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n}\n\n.anticon > * {\n line-height: 1;\n}\n\n.anticon svg {\n display: inline-block;\n}\n\n.anticon::before {\n display: none;\n}\n\n.anticon .anticon-icon {\n display: block;\n}\n\n.anticon[tabindex] {\n cursor: pointer;\n}\n\n.anticon-spin::before,\n.anticon-spin {\n display: inline-block;\n -webkit-animation: loadingCircle 1s infinite linear;\n animation: loadingCircle 1s infinite linear;\n}\n\n@-webkit-keyframes loadingCircle {\n 100% {\n -webkit-transform: rotate(360deg);\n transform: rotate(360deg);\n }\n}\n\n@keyframes loadingCircle {\n 100% {\n -webkit-transform: rotate(360deg);\n transform: rotate(360deg);\n }\n}\n";r&&(o=o.replace(/anticon/g,r)),(0,l.useEffect)(function(){var t=e.current,r=(0,g.A)(t);(0,m.hq)(o,"@ant-design-icons",{prepend:!0,csp:n,attachTo:r})},[])},w=["icon","className","onClick","style","primaryColor","secondaryColor"],x={primaryColor:"#333",secondaryColor:"#E6E6E6",calculated:!1},O=function(e){var t,n,r=e.icon,o=e.className,a=e.onClick,s=e.style,c=e.primaryColor,u=e.secondaryColor,d=(0,i.Z)(e,w),f=l.useRef(),m=x;if(c&&(m={primaryColor:c,secondaryColor:u||y(c)}),S(f),t=b(r),n="icon should be icon definiton, but got ".concat(r),(0,h.ZP)(t,"[@ant-design/icons] ".concat(n)),!b(r))return null;var g=r;return g&&"function"==typeof g.icon&&(g=(0,p.Z)((0,p.Z)({},g),{},{icon:g.icon(m.primaryColor,m.secondaryColor)})),function e(t,n,r){return r?l.createElement(t.tag,(0,p.Z)((0,p.Z)({key:n},v(t.attrs)),r),(t.children||[]).map(function(r,o){return e(r,"".concat(n,"-").concat(t.tag,"-").concat(o))})):l.createElement(t.tag,(0,p.Z)({key:n},v(t.attrs)),(t.children||[]).map(function(r,o){return e(r,"".concat(n,"-").concat(t.tag,"-").concat(o))}))}(g.icon,"svg-".concat(g.name),(0,p.Z)((0,p.Z)({className:o,onClick:a,style:s,"data-icon":g.name,width:"1em",height:"1em",fill:"currentColor","aria-hidden":"true"},d),{},{ref:f}))};function k(e){var t=E(e),n=(0,o.Z)(t,2),r=n[0],a=n[1];return O.setTwoToneColors({primaryColor:r,secondaryColor:a})}O.displayName="IconReact",O.getTwoToneColors=function(){return(0,p.Z)({},x)},O.setTwoToneColors=function(e){var t=e.primaryColor,n=e.secondaryColor;x.primaryColor=t,x.secondaryColor=n||y(t),x.calculated=!!n};var C=["className","icon","spin","rotate","tabIndex","onClick","twoToneColor"];k(u.iN.primary);var A=l.forwardRef(function(e,t){var n,s=e.className,u=e.icon,p=e.spin,f=e.rotate,m=e.tabIndex,g=e.onClick,h=e.twoToneColor,b=(0,i.Z)(e,C),v=l.useContext(d.Z),y=v.prefixCls,S=void 0===y?"anticon":y,w=v.rootClassName,x=c()(w,S,(n={},(0,a.Z)(n,"".concat(S,"-").concat(u.name),!!u.name),(0,a.Z)(n,"".concat(S,"-spin"),!!p||"loading"===u.name),n),s),k=m;void 0===k&&g&&(k=-1);var A=E(h),T=(0,o.Z)(A,2),I=T[0],N=T[1];return l.createElement("span",(0,r.Z)({role:"img","aria-label":u.name},b,{ref:t,tabIndex:k,onClick:g,className:x}),l.createElement(O,{icon:u,primaryColor:I,secondaryColor:N,style:f?{msTransform:"rotate(".concat(f,"deg)"),transform:"rotate(".concat(f,"deg)")}:void 0}))});A.displayName="AntdIcon",A.getTwoToneColor=function(){var e=O.getTwoToneColors();return e.calculated?[e.primaryColor,e.secondaryColor]:e.primaryColor},A.setTwoToneColor=k;var T=A},67689:function(e,t,n){var r=(0,n(64090).createContext)({});t.Z=r},99537:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm193.5 301.7l-210.6 292a31.8 31.8 0 01-51.7 0L318.5 484.9c-3.8-5.3 0-12.7 6.5-12.7h46.9c10.2 0 19.9 4.9 25.9 13.3l71.2 98.8 157.2-218c6-8.3 15.6-13.3 25.9-13.3H699c6.5 0 10.3 7.4 6.5 12.7z"}}]},name:"check-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},90507:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M912 190h-69.9c-9.8 0-19.1 4.5-25.1 12.2L404.7 724.5 207 474a32 32 0 00-25.1-12.2H112c-6.7 0-10.4 7.7-6.3 12.9l273.9 347c12.8 16.2 37.4 16.2 50.3 0l488.4-618.9c4.1-5.1.4-12.8-6.3-12.8z"}}]},name:"check",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},77136:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{"fill-rule":"evenodd",viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64c247.4 0 448 200.6 448 448S759.4 960 512 960 64 759.4 64 512 264.6 64 512 64zm127.98 274.82h-.04l-.08.06L512 466.75 384.14 338.88c-.04-.05-.06-.06-.08-.06a.12.12 0 00-.07 0c-.03 0-.05.01-.09.05l-45.02 45.02a.2.2 0 00-.05.09.12.12 0 000 .07v.02a.27.27 0 00.06.06L466.75 512 338.88 639.86c-.05.04-.06.06-.06.08a.12.12 0 000 .07c0 .03.01.05.05.09l45.02 45.02a.2.2 0 00.09.05.12.12 0 00.07 0c.02 0 .04-.01.08-.05L512 557.25l127.86 127.87c.04.04.06.05.08.05a.12.12 0 00.07 0c.03 0 .05-.01.09-.05l45.02-45.02a.2.2 0 00.05-.09.12.12 0 000-.07v-.02a.27.27 0 00-.05-.06L557.25 512l127.87-127.86c.04-.04.05-.06.05-.08a.12.12 0 000-.07c0-.03-.01-.05-.05-.09l-45.02-45.02a.2.2 0 00-.09-.05.12.12 0 00-.07 0z"}}]},name:"close-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},81303:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{"fill-rule":"evenodd",viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M799.86 166.31c.02 0 .04.02.08.06l57.69 57.7c.04.03.05.05.06.08a.12.12 0 010 .06c0 .03-.02.05-.06.09L569.93 512l287.7 287.7c.04.04.05.06.06.09a.12.12 0 010 .07c0 .02-.02.04-.06.08l-57.7 57.69c-.03.04-.05.05-.07.06a.12.12 0 01-.07 0c-.03 0-.05-.02-.09-.06L512 569.93l-287.7 287.7c-.04.04-.06.05-.09.06a.12.12 0 01-.07 0c-.02 0-.04-.02-.08-.06l-57.69-57.7c-.04-.03-.05-.05-.06-.07a.12.12 0 010-.07c0-.03.02-.05.06-.09L454.07 512l-287.7-287.7c-.04-.04-.05-.06-.06-.09a.12.12 0 010-.07c0-.02.02-.04.06-.08l57.7-57.69c.03-.04.05-.05.07-.06a.12.12 0 01.07 0c.03 0 .05.02.09.06L512 454.07l287.7-287.7c.04-.04.06-.05.09-.06a.12.12 0 01.07 0z"}}]},name:"close",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},20383:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M884 256h-75c-5.1 0-9.9 2.5-12.9 6.6L512 654.2 227.9 262.6c-3-4.1-7.8-6.6-12.9-6.6h-75c-6.5 0-10.3 7.4-6.5 12.7l352.6 486.1c12.8 17.6 39 17.6 51.7 0l352.6-486.1c3.9-5.3.1-12.7-6.4-12.7z"}}]},name:"down",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},31413:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M176 511a56 56 0 10112 0 56 56 0 10-112 0zm280 0a56 56 0 10112 0 56 56 0 10-112 0zm280 0a56 56 0 10112 0 56 56 0 10-112 0z"}}]},name:"ellipsis",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},20653:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm-32 232c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v272c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8V296zm32 440a48.01 48.01 0 010-96 48.01 48.01 0 010 96z"}}]},name:"exclamation-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},41311:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M942.2 486.2C847.4 286.5 704.1 186 512 186c-192.2 0-335.4 100.5-430.2 300.3a60.3 60.3 0 000 51.5C176.6 737.5 319.9 838 512 838c192.2 0 335.4-100.5 430.2-300.3 7.7-16.2 7.7-35 0-51.5zM512 766c-161.3 0-279.4-81.8-362.7-254C232.6 339.8 350.7 258 512 258c161.3 0 279.4 81.8 362.7 254C791.5 684.2 673.4 766 512 766zm-4-430c-97.2 0-176 78.8-176 176s78.8 176 176 176 176-78.8 176-176-78.8-176-176-176zm0 288c-61.9 0-112-50.1-112-112s50.1-112 112-112 112 50.1 112 112-50.1 112-112 112z"}}]},name:"eye",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},40388:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm32 664c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8V456c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v272zm-32-344a48.01 48.01 0 010-96 48.01 48.01 0 010 96z"}}]},name:"info-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},66155:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"0 0 1024 1024",focusable:"false"},children:[{tag:"path",attrs:{d:"M988 548c-19.9 0-36-16.1-36-36 0-59.4-11.6-117-34.6-171.3a440.45 440.45 0 00-94.3-139.9 437.71 437.71 0 00-139.9-94.3C629 83.6 571.4 72 512 72c-19.9 0-36-16.1-36-36s16.1-36 36-36c69.1 0 136.2 13.5 199.3 40.3C772.3 66 827 103 874 150c47 47 83.9 101.8 109.7 162.7 26.7 63.1 40.2 130.2 40.2 199.3.1 19.9-16 36-35.9 36z"}}]},name:"loading",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},50459:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M765.7 486.8L314.9 134.7A7.97 7.97 0 00302 141v77.3c0 4.9 2.3 9.6 6.1 12.6l360 281.1-360 281.1c-3.9 3-6.1 7.7-6.1 12.6V883c0 6.7 7.7 10.4 12.9 6.3l450.8-352.1a31.96 31.96 0 000-50.4z"}}]},name:"right",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},96871:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M909.6 854.5L649.9 594.8C690.2 542.7 712 479 712 412c0-80.2-31.3-155.4-87.9-212.1-56.6-56.7-132-87.9-212.1-87.9s-155.5 31.3-212.1 87.9C143.2 256.5 112 331.8 112 412c0 80.1 31.3 155.5 87.9 212.1C256.5 680.8 331.8 712 412 712c67 0 130.6-21.8 182.7-62l259.7 259.6a8.2 8.2 0 0011.6 0l43.6-43.5a8.2 8.2 0 000-11.6zM570.4 570.4C528 612.7 471.8 636 412 636s-116-23.3-158.4-65.6C211.3 528 188 471.8 188 412s23.3-116.1 65.6-158.4C296 211.3 352.2 188 412 188s116.1 23.2 158.4 65.6S636 352.2 636 412s-23.3 116.1-65.6 158.4z"}}]},name:"search",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},97766:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M400 317.7h73.9V656c0 4.4 3.6 8 8 8h60c4.4 0 8-3.6 8-8V317.7H624c6.7 0 10.4-7.7 6.3-12.9L518.3 163a8 8 0 00-12.6 0l-112 141.7c-4.1 5.3-.4 13 6.3 13zM878 626h-60c-4.4 0-8 3.6-8 8v154H214V634c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v198c0 17.7 14.3 32 32 32h684c17.7 0 32-14.3 32-32V634c0-4.4-3.6-8-8-8z"}}]},name:"upload",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},41785:function(e,t,n){n.d(t,{T6:function(){return p},VD:function(){return f},WE:function(){return c},Yt:function(){return m},lC:function(){return a},py:function(){return s},rW:function(){return o},s:function(){return d},ve:function(){return l},vq:function(){return u}});var r=n(27974);function o(e,t,n){return{r:255*(0,r.sh)(e,255),g:255*(0,r.sh)(t,255),b:255*(0,r.sh)(n,255)}}function a(e,t,n){var o=Math.max(e=(0,r.sh)(e,255),t=(0,r.sh)(t,255),n=(0,r.sh)(n,255)),a=Math.min(e,t,n),i=0,l=0,s=(o+a)/2;if(o===a)l=0,i=0;else{var c=o-a;switch(l=s>.5?c/(2-o-a):c/(o+a),o){case e:i=(t-n)/c+(t1&&(n-=1),n<1/6)?e+6*n*(t-e):n<.5?t:n<2/3?e+(t-e)*(2/3-n)*6:e}function l(e,t,n){if(e=(0,r.sh)(e,360),t=(0,r.sh)(t,100),n=(0,r.sh)(n,100),0===t)a=n,l=n,o=n;else{var o,a,l,s=n<.5?n*(1+t):n+t-n*t,c=2*n-s;o=i(c,s,e+1/3),a=i(c,s,e),l=i(c,s,e-1/3)}return{r:255*o,g:255*a,b:255*l}}function s(e,t,n){var o=Math.max(e=(0,r.sh)(e,255),t=(0,r.sh)(t,255),n=(0,r.sh)(n,255)),a=Math.min(e,t,n),i=0,l=o-a;if(o===a)i=0;else{switch(o){case e:i=(t-n)/l+(t>16,g:(65280&e)>>8,b:255&e}}},6564:function(e,t,n){n.d(t,{R:function(){return r}});var r={aliceblue:"#f0f8ff",antiquewhite:"#faebd7",aqua:"#00ffff",aquamarine:"#7fffd4",azure:"#f0ffff",beige:"#f5f5dc",bisque:"#ffe4c4",black:"#000000",blanchedalmond:"#ffebcd",blue:"#0000ff",blueviolet:"#8a2be2",brown:"#a52a2a",burlywood:"#deb887",cadetblue:"#5f9ea0",chartreuse:"#7fff00",chocolate:"#d2691e",coral:"#ff7f50",cornflowerblue:"#6495ed",cornsilk:"#fff8dc",crimson:"#dc143c",cyan:"#00ffff",darkblue:"#00008b",darkcyan:"#008b8b",darkgoldenrod:"#b8860b",darkgray:"#a9a9a9",darkgreen:"#006400",darkgrey:"#a9a9a9",darkkhaki:"#bdb76b",darkmagenta:"#8b008b",darkolivegreen:"#556b2f",darkorange:"#ff8c00",darkorchid:"#9932cc",darkred:"#8b0000",darksalmon:"#e9967a",darkseagreen:"#8fbc8f",darkslateblue:"#483d8b",darkslategray:"#2f4f4f",darkslategrey:"#2f4f4f",darkturquoise:"#00ced1",darkviolet:"#9400d3",deeppink:"#ff1493",deepskyblue:"#00bfff",dimgray:"#696969",dimgrey:"#696969",dodgerblue:"#1e90ff",firebrick:"#b22222",floralwhite:"#fffaf0",forestgreen:"#228b22",fuchsia:"#ff00ff",gainsboro:"#dcdcdc",ghostwhite:"#f8f8ff",goldenrod:"#daa520",gold:"#ffd700",gray:"#808080",green:"#008000",greenyellow:"#adff2f",grey:"#808080",honeydew:"#f0fff0",hotpink:"#ff69b4",indianred:"#cd5c5c",indigo:"#4b0082",ivory:"#fffff0",khaki:"#f0e68c",lavenderblush:"#fff0f5",lavender:"#e6e6fa",lawngreen:"#7cfc00",lemonchiffon:"#fffacd",lightblue:"#add8e6",lightcoral:"#f08080",lightcyan:"#e0ffff",lightgoldenrodyellow:"#fafad2",lightgray:"#d3d3d3",lightgreen:"#90ee90",lightgrey:"#d3d3d3",lightpink:"#ffb6c1",lightsalmon:"#ffa07a",lightseagreen:"#20b2aa",lightskyblue:"#87cefa",lightslategray:"#778899",lightslategrey:"#778899",lightsteelblue:"#b0c4de",lightyellow:"#ffffe0",lime:"#00ff00",limegreen:"#32cd32",linen:"#faf0e6",magenta:"#ff00ff",maroon:"#800000",mediumaquamarine:"#66cdaa",mediumblue:"#0000cd",mediumorchid:"#ba55d3",mediumpurple:"#9370db",mediumseagreen:"#3cb371",mediumslateblue:"#7b68ee",mediumspringgreen:"#00fa9a",mediumturquoise:"#48d1cc",mediumvioletred:"#c71585",midnightblue:"#191970",mintcream:"#f5fffa",mistyrose:"#ffe4e1",moccasin:"#ffe4b5",navajowhite:"#ffdead",navy:"#000080",oldlace:"#fdf5e6",olive:"#808000",olivedrab:"#6b8e23",orange:"#ffa500",orangered:"#ff4500",orchid:"#da70d6",palegoldenrod:"#eee8aa",palegreen:"#98fb98",paleturquoise:"#afeeee",palevioletred:"#db7093",papayawhip:"#ffefd5",peachpuff:"#ffdab9",peru:"#cd853f",pink:"#ffc0cb",plum:"#dda0dd",powderblue:"#b0e0e6",purple:"#800080",rebeccapurple:"#663399",red:"#ff0000",rosybrown:"#bc8f8f",royalblue:"#4169e1",saddlebrown:"#8b4513",salmon:"#fa8072",sandybrown:"#f4a460",seagreen:"#2e8b57",seashell:"#fff5ee",sienna:"#a0522d",silver:"#c0c0c0",skyblue:"#87ceeb",slateblue:"#6a5acd",slategray:"#708090",slategrey:"#708090",snow:"#fffafa",springgreen:"#00ff7f",steelblue:"#4682b4",tan:"#d2b48c",teal:"#008080",thistle:"#d8bfd8",tomato:"#ff6347",turquoise:"#40e0d0",violet:"#ee82ee",wheat:"#f5deb3",white:"#ffffff",whitesmoke:"#f5f5f5",yellow:"#ffff00",yellowgreen:"#9acd32"}},76991:function(e,t,n){n.d(t,{uA:function(){return i}});var r=n(41785),o=n(6564),a=n(27974);function i(e){var t={r:0,g:0,b:0},n=1,i=null,l=null,s=null,c=!1,p=!1;return"string"==typeof e&&(e=function(e){if(0===(e=e.trim().toLowerCase()).length)return!1;var t=!1;if(o.R[e])e=o.R[e],t=!0;else if("transparent"===e)return{r:0,g:0,b:0,a:0,format:"name"};var n=u.rgb.exec(e);return n?{r:n[1],g:n[2],b:n[3]}:(n=u.rgba.exec(e))?{r:n[1],g:n[2],b:n[3],a:n[4]}:(n=u.hsl.exec(e))?{h:n[1],s:n[2],l:n[3]}:(n=u.hsla.exec(e))?{h:n[1],s:n[2],l:n[3],a:n[4]}:(n=u.hsv.exec(e))?{h:n[1],s:n[2],v:n[3]}:(n=u.hsva.exec(e))?{h:n[1],s:n[2],v:n[3],a:n[4]}:(n=u.hex8.exec(e))?{r:(0,r.VD)(n[1]),g:(0,r.VD)(n[2]),b:(0,r.VD)(n[3]),a:(0,r.T6)(n[4]),format:t?"name":"hex8"}:(n=u.hex6.exec(e))?{r:(0,r.VD)(n[1]),g:(0,r.VD)(n[2]),b:(0,r.VD)(n[3]),format:t?"name":"hex"}:(n=u.hex4.exec(e))?{r:(0,r.VD)(n[1]+n[1]),g:(0,r.VD)(n[2]+n[2]),b:(0,r.VD)(n[3]+n[3]),a:(0,r.T6)(n[4]+n[4]),format:t?"name":"hex8"}:!!(n=u.hex3.exec(e))&&{r:(0,r.VD)(n[1]+n[1]),g:(0,r.VD)(n[2]+n[2]),b:(0,r.VD)(n[3]+n[3]),format:t?"name":"hex"}}(e)),"object"==typeof e&&(d(e.r)&&d(e.g)&&d(e.b)?(t=(0,r.rW)(e.r,e.g,e.b),c=!0,p="%"===String(e.r).substr(-1)?"prgb":"rgb"):d(e.h)&&d(e.s)&&d(e.v)?(i=(0,a.JX)(e.s),l=(0,a.JX)(e.v),t=(0,r.WE)(e.h,i,l),c=!0,p="hsv"):d(e.h)&&d(e.s)&&d(e.l)&&(i=(0,a.JX)(e.s),s=(0,a.JX)(e.l),t=(0,r.ve)(e.h,i,s),c=!0,p="hsl"),Object.prototype.hasOwnProperty.call(e,"a")&&(n=e.a)),n=(0,a.Yq)(n),{ok:c,format:e.format||p,r:Math.min(255,Math.max(t.r,0)),g:Math.min(255,Math.max(t.g,0)),b:Math.min(255,Math.max(t.b,0)),a:n}}var l="(?:".concat("[-\\+]?\\d*\\.\\d+%?",")|(?:").concat("[-\\+]?\\d+%?",")"),s="[\\s|\\(]+(".concat(l,")[,|\\s]+(").concat(l,")[,|\\s]+(").concat(l,")\\s*\\)?"),c="[\\s|\\(]+(".concat(l,")[,|\\s]+(").concat(l,")[,|\\s]+(").concat(l,")[,|\\s]+(").concat(l,")\\s*\\)?"),u={CSS_UNIT:new RegExp(l),rgb:RegExp("rgb"+s),rgba:RegExp("rgba"+c),hsl:RegExp("hsl"+s),hsla:RegExp("hsla"+c),hsv:RegExp("hsv"+s),hsva:RegExp("hsva"+c),hex3:/^#?([0-9a-fA-F]{1})([0-9a-fA-F]{1})([0-9a-fA-F]{1})$/,hex6:/^#?([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})$/,hex4:/^#?([0-9a-fA-F]{1})([0-9a-fA-F]{1})([0-9a-fA-F]{1})([0-9a-fA-F]{1})$/,hex8:/^#?([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})$/};function d(e){return!!u.CSS_UNIT.exec(String(e))}},6336:function(e,t,n){n.d(t,{C:function(){return l}});var r=n(41785),o=n(6564),a=n(76991),i=n(27974),l=function(){function e(t,n){if(void 0===t&&(t=""),void 0===n&&(n={}),t instanceof e)return t;"number"==typeof t&&(t=(0,r.Yt)(t)),this.originalInput=t;var o,i=(0,a.uA)(t);this.originalInput=t,this.r=i.r,this.g=i.g,this.b=i.b,this.a=i.a,this.roundA=Math.round(100*this.a)/100,this.format=null!==(o=n.format)&&void 0!==o?o:i.format,this.gradientType=n.gradientType,this.r<1&&(this.r=Math.round(this.r)),this.g<1&&(this.g=Math.round(this.g)),this.b<1&&(this.b=Math.round(this.b)),this.isValid=i.ok}return e.prototype.isDark=function(){return 128>this.getBrightness()},e.prototype.isLight=function(){return!this.isDark()},e.prototype.getBrightness=function(){var e=this.toRgb();return(299*e.r+587*e.g+114*e.b)/1e3},e.prototype.getLuminance=function(){var e=this.toRgb(),t=e.r/255,n=e.g/255,r=e.b/255;return .2126*(t<=.03928?t/12.92:Math.pow((t+.055)/1.055,2.4))+.7152*(n<=.03928?n/12.92:Math.pow((n+.055)/1.055,2.4))+.0722*(r<=.03928?r/12.92:Math.pow((r+.055)/1.055,2.4))},e.prototype.getAlpha=function(){return this.a},e.prototype.setAlpha=function(e){return this.a=(0,i.Yq)(e),this.roundA=Math.round(100*this.a)/100,this},e.prototype.isMonochrome=function(){return 0===this.toHsl().s},e.prototype.toHsv=function(){var e=(0,r.py)(this.r,this.g,this.b);return{h:360*e.h,s:e.s,v:e.v,a:this.a}},e.prototype.toHsvString=function(){var e=(0,r.py)(this.r,this.g,this.b),t=Math.round(360*e.h),n=Math.round(100*e.s),o=Math.round(100*e.v);return 1===this.a?"hsv(".concat(t,", ").concat(n,"%, ").concat(o,"%)"):"hsva(".concat(t,", ").concat(n,"%, ").concat(o,"%, ").concat(this.roundA,")")},e.prototype.toHsl=function(){var e=(0,r.lC)(this.r,this.g,this.b);return{h:360*e.h,s:e.s,l:e.l,a:this.a}},e.prototype.toHslString=function(){var e=(0,r.lC)(this.r,this.g,this.b),t=Math.round(360*e.h),n=Math.round(100*e.s),o=Math.round(100*e.l);return 1===this.a?"hsl(".concat(t,", ").concat(n,"%, ").concat(o,"%)"):"hsla(".concat(t,", ").concat(n,"%, ").concat(o,"%, ").concat(this.roundA,")")},e.prototype.toHex=function(e){return void 0===e&&(e=!1),(0,r.vq)(this.r,this.g,this.b,e)},e.prototype.toHexString=function(e){return void 0===e&&(e=!1),"#"+this.toHex(e)},e.prototype.toHex8=function(e){return void 0===e&&(e=!1),(0,r.s)(this.r,this.g,this.b,this.a,e)},e.prototype.toHex8String=function(e){return void 0===e&&(e=!1),"#"+this.toHex8(e)},e.prototype.toHexShortString=function(e){return void 0===e&&(e=!1),1===this.a?this.toHexString(e):this.toHex8String(e)},e.prototype.toRgb=function(){return{r:Math.round(this.r),g:Math.round(this.g),b:Math.round(this.b),a:this.a}},e.prototype.toRgbString=function(){var e=Math.round(this.r),t=Math.round(this.g),n=Math.round(this.b);return 1===this.a?"rgb(".concat(e,", ").concat(t,", ").concat(n,")"):"rgba(".concat(e,", ").concat(t,", ").concat(n,", ").concat(this.roundA,")")},e.prototype.toPercentageRgb=function(){var e=function(e){return"".concat(Math.round(100*(0,i.sh)(e,255)),"%")};return{r:e(this.r),g:e(this.g),b:e(this.b),a:this.a}},e.prototype.toPercentageRgbString=function(){var e=function(e){return Math.round(100*(0,i.sh)(e,255))};return 1===this.a?"rgb(".concat(e(this.r),"%, ").concat(e(this.g),"%, ").concat(e(this.b),"%)"):"rgba(".concat(e(this.r),"%, ").concat(e(this.g),"%, ").concat(e(this.b),"%, ").concat(this.roundA,")")},e.prototype.toName=function(){if(0===this.a)return"transparent";if(this.a<1)return!1;for(var e="#"+(0,r.vq)(this.r,this.g,this.b,!1),t=0,n=Object.entries(o.R);t=0;return!t&&r&&(e.startsWith("hex")||"name"===e)?"name"===e&&0===this.a?this.toName():this.toRgbString():("rgb"===e&&(n=this.toRgbString()),"prgb"===e&&(n=this.toPercentageRgbString()),("hex"===e||"hex6"===e)&&(n=this.toHexString()),"hex3"===e&&(n=this.toHexString(!0)),"hex4"===e&&(n=this.toHex8String(!0)),"hex8"===e&&(n=this.toHex8String()),"name"===e&&(n=this.toName()),"hsl"===e&&(n=this.toHslString()),"hsv"===e&&(n=this.toHsvString()),n||this.toHexString())},e.prototype.toNumber=function(){return(Math.round(this.r)<<16)+(Math.round(this.g)<<8)+Math.round(this.b)},e.prototype.clone=function(){return new e(this.toString())},e.prototype.lighten=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.l+=t/100,n.l=(0,i.V2)(n.l),new e(n)},e.prototype.brighten=function(t){void 0===t&&(t=10);var n=this.toRgb();return n.r=Math.max(0,Math.min(255,n.r-Math.round(-(t/100*255)))),n.g=Math.max(0,Math.min(255,n.g-Math.round(-(t/100*255)))),n.b=Math.max(0,Math.min(255,n.b-Math.round(-(t/100*255)))),new e(n)},e.prototype.darken=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.l-=t/100,n.l=(0,i.V2)(n.l),new e(n)},e.prototype.tint=function(e){return void 0===e&&(e=10),this.mix("white",e)},e.prototype.shade=function(e){return void 0===e&&(e=10),this.mix("black",e)},e.prototype.desaturate=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.s-=t/100,n.s=(0,i.V2)(n.s),new e(n)},e.prototype.saturate=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.s+=t/100,n.s=(0,i.V2)(n.s),new e(n)},e.prototype.greyscale=function(){return this.desaturate(100)},e.prototype.spin=function(t){var n=this.toHsl(),r=(n.h+t)%360;return n.h=r<0?360+r:r,new e(n)},e.prototype.mix=function(t,n){void 0===n&&(n=50);var r=this.toRgb(),o=new e(t).toRgb(),a=n/100;return new e({r:(o.r-r.r)*a+r.r,g:(o.g-r.g)*a+r.g,b:(o.b-r.b)*a+r.b,a:(o.a-r.a)*a+r.a})},e.prototype.analogous=function(t,n){void 0===t&&(t=6),void 0===n&&(n=30);var r=this.toHsl(),o=360/n,a=[this];for(r.h=(r.h-(o*t>>1)+720)%360;--t;)r.h=(r.h+o)%360,a.push(new e(r));return a},e.prototype.complement=function(){var t=this.toHsl();return t.h=(t.h+180)%360,new e(t)},e.prototype.monochromatic=function(t){void 0===t&&(t=6);for(var n=this.toHsv(),r=n.h,o=n.s,a=n.v,i=[],l=1/t;t--;)i.push(new e({h:r,s:o,v:a})),a=(a+l)%1;return i},e.prototype.splitcomplement=function(){var t=this.toHsl(),n=t.h;return[this,new e({h:(n+72)%360,s:t.s,l:t.l}),new e({h:(n+216)%360,s:t.s,l:t.l})]},e.prototype.onBackground=function(t){var n=this.toRgb(),r=new e(t).toRgb(),o=n.a+r.a*(1-n.a);return new e({r:(n.r*n.a+r.r*r.a*(1-n.a))/o,g:(n.g*n.a+r.g*r.a*(1-n.a))/o,b:(n.b*n.a+r.b*r.a*(1-n.a))/o,a:o})},e.prototype.triad=function(){return this.polyad(3)},e.prototype.tetrad=function(){return this.polyad(4)},e.prototype.polyad=function(t){for(var n=this.toHsl(),r=n.h,o=[this],a=360/t,i=1;iMath.abs(e-t))?1:e=360===t?(e<0?e%t+t:e%t)/parseFloat(String(t)):e%t/parseFloat(String(t))}function o(e){return Math.min(1,Math.max(0,e))}function a(e){return(isNaN(e=parseFloat(e))||e<0||e>1)&&(e=1),e}function i(e){return e<=1?"".concat(100*Number(e),"%"):e}function l(e){return 1===e.length?"0"+e:String(e)}n.d(t,{FZ:function(){return l},JX:function(){return i},V2:function(){return o},Yq:function(){return a},sh:function(){return r}})},88804:function(e,t,n){n.d(t,{Z:function(){return y}});var r,o=n(80406),a=n(64090),i=n(89542),l=n(22127);n(53850);var s=n(74084),c=a.createContext(null),u=n(63787),d=n(24800),p=[],f=n(24050);function m(e){var t=e.match(/^(.*)px$/),n=Number(null==t?void 0:t[1]);return Number.isNaN(n)?function(e){if("undefined"==typeof document)return 0;if(void 0===r){var t=document.createElement("div");t.style.width="100%",t.style.height="200px";var n=document.createElement("div"),o=n.style;o.position="absolute",o.top="0",o.left="0",o.pointerEvents="none",o.visibility="hidden",o.width="200px",o.height="150px",o.overflow="hidden",n.appendChild(t),document.body.appendChild(n);var a=t.offsetWidth;n.style.overflow="scroll";var i=t.offsetWidth;a===i&&(i=n.clientWidth),document.body.removeChild(n),r=a-i}return r}():n}var g="rc-util-locker-".concat(Date.now()),h=0,b=!1,v=function(e){return!1!==e&&((0,l.Z)()&&e?"string"==typeof e?document.querySelector(e):"function"==typeof e?e():e:null)},y=a.forwardRef(function(e,t){var n,r,y,E,S=e.open,w=e.autoLock,x=e.getContainer,O=(e.debug,e.autoDestroy),k=void 0===O||O,C=e.children,A=a.useState(S),T=(0,o.Z)(A,2),I=T[0],N=T[1],R=I||S;a.useEffect(function(){(k||S)&&N(S)},[S,k]);var _=a.useState(function(){return v(x)}),P=(0,o.Z)(_,2),L=P[0],M=P[1];a.useEffect(function(){var e=v(x);M(null!=e?e:null)});var D=function(e,t){var n=a.useState(function(){return(0,l.Z)()?document.createElement("div"):null}),r=(0,o.Z)(n,1)[0],i=a.useRef(!1),s=a.useContext(c),f=a.useState(p),m=(0,o.Z)(f,2),g=m[0],h=m[1],b=s||(i.current?void 0:function(e){h(function(t){return[e].concat((0,u.Z)(t))})});function v(){r.parentElement||document.body.appendChild(r),i.current=!0}function y(){var e;null===(e=r.parentElement)||void 0===e||e.removeChild(r),i.current=!1}return(0,d.Z)(function(){return e?s?s(v):v():y(),y},[e]),(0,d.Z)(function(){g.length&&(g.forEach(function(e){return e()}),h(p))},[g]),[r,b]}(R&&!L,0),j=(0,o.Z)(D,2),F=j[0],B=j[1],Z=null!=L?L:F;n=!!(w&&S&&(0,l.Z)()&&(Z===F||Z===document.body)),r=a.useState(function(){return h+=1,"".concat(g,"_").concat(h)}),y=(0,o.Z)(r,1)[0],(0,d.Z)(function(){if(n){var e=function(e){if("undefined"==typeof document||!e||!(e instanceof Element))return{width:0,height:0};var t=getComputedStyle(e,"::-webkit-scrollbar"),n=t.width,r=t.height;return{width:m(n),height:m(r)}}(document.body).width,t=document.body.scrollHeight>(window.innerHeight||document.documentElement.clientHeight)&&window.innerWidth>document.body.offsetWidth;(0,f.hq)("\nhtml body {\n overflow-y: hidden;\n ".concat(t?"width: calc(100% - ".concat(e,"px);"):"","\n}"),y)}else(0,f.jL)(y);return function(){(0,f.jL)(y)}},[n,y]);var U=null;C&&(0,s.Yr)(C)&&t&&(U=C.ref);var z=(0,s.x1)(U,t);if(!R||!(0,l.Z)()||void 0===L)return null;var H=!1===Z||("boolean"==typeof E&&(b=E),b),G=C;return t&&(G=a.cloneElement(C,{ref:z})),a.createElement(c.Provider,{value:B},H?G:(0,i.createPortal)(G,Z))})},44101:function(e,t,n){n.d(t,{Z:function(){return z}});var r=n(5239),o=n(80406),a=n(6787),i=n(88804),l=n(16480),s=n.n(l),c=n(46505),u=n(97472),d=n(74687),p=n(54811),f=n(91010),m=n(24800),g=n(76158),h=n(64090),b=n(14749),v=n(49367),y=n(74084);function E(e){var t=e.prefixCls,n=e.align,r=e.arrow,o=e.arrowPos,a=r||{},i=a.className,l=a.content,c=o.x,u=o.y,d=h.useRef();if(!n||!n.points)return null;var p={position:"absolute"};if(!1!==n.autoArrow){var f=n.points[0],m=n.points[1],g=f[0],b=f[1],v=m[0],y=m[1];g!==v&&["t","b"].includes(g)?"t"===g?p.top=0:p.bottom=0:p.top=void 0===u?0:u,b!==y&&["l","r"].includes(b)?"l"===b?p.left=0:p.right=0:p.left=void 0===c?0:c}return h.createElement("div",{ref:d,className:s()("".concat(t,"-arrow"),i),style:p},l)}function S(e){var t=e.prefixCls,n=e.open,r=e.zIndex,o=e.mask,a=e.motion;return o?h.createElement(v.ZP,(0,b.Z)({},a,{motionAppear:!0,visible:n,removeOnLeave:!0}),function(e){var n=e.className;return h.createElement("div",{style:{zIndex:r},className:s()("".concat(t,"-mask"),n)})}):null}var w=h.memo(function(e){return e.children},function(e,t){return t.cache}),x=h.forwardRef(function(e,t){var n=e.popup,a=e.className,i=e.prefixCls,l=e.style,u=e.target,d=e.onVisibleChanged,p=e.open,f=e.keepDom,g=e.fresh,x=e.onClick,O=e.mask,k=e.arrow,C=e.arrowPos,A=e.align,T=e.motion,I=e.maskMotion,N=e.forceRender,R=e.getPopupContainer,_=e.autoDestroy,P=e.portal,L=e.zIndex,M=e.onMouseEnter,D=e.onMouseLeave,j=e.onPointerEnter,F=e.ready,B=e.offsetX,Z=e.offsetY,U=e.offsetR,z=e.offsetB,H=e.onAlign,G=e.onPrepare,$=e.stretch,W=e.targetWidth,V=e.targetHeight,q="function"==typeof n?n():n,Y=p||f,K=(null==R?void 0:R.length)>0,X=h.useState(!R||!K),Q=(0,o.Z)(X,2),J=Q[0],ee=Q[1];if((0,m.Z)(function(){!J&&K&&u&&ee(!0)},[J,K,u]),!J)return null;var et="auto",en={left:"-1000vw",top:"-1000vh",right:et,bottom:et};if(F||!p){var er,eo=A.points,ea=A.dynamicInset||(null===(er=A._experimental)||void 0===er?void 0:er.dynamicInset),ei=ea&&"r"===eo[0][1],el=ea&&"b"===eo[0][0];ei?(en.right=U,en.left=et):(en.left=B,en.right=et),el?(en.bottom=z,en.top=et):(en.top=Z,en.bottom=et)}var es={};return $&&($.includes("height")&&V?es.height=V:$.includes("minHeight")&&V&&(es.minHeight=V),$.includes("width")&&W?es.width=W:$.includes("minWidth")&&W&&(es.minWidth=W)),p||(es.pointerEvents="none"),h.createElement(P,{open:N||Y,getContainer:R&&function(){return R(u)},autoDestroy:_},h.createElement(S,{prefixCls:i,open:p,zIndex:L,mask:O,motion:I}),h.createElement(c.Z,{onResize:H,disabled:!p},function(e){return h.createElement(v.ZP,(0,b.Z)({motionAppear:!0,motionEnter:!0,motionLeave:!0,removeOnLeave:!1,forceRender:N,leavedClassName:"".concat(i,"-hidden")},T,{onAppearPrepare:G,onEnterPrepare:G,visible:p,onVisibleChanged:function(e){var t;null==T||null===(t=T.onVisibleChanged)||void 0===t||t.call(T,e),d(e)}}),function(n,o){var c=n.className,u=n.style,d=s()(i,c,a);return h.createElement("div",{ref:(0,y.sQ)(e,t,o),className:d,style:(0,r.Z)((0,r.Z)((0,r.Z)((0,r.Z)({"--arrow-x":"".concat(C.x||0,"px"),"--arrow-y":"".concat(C.y||0,"px")},en),es),u),{},{boxSizing:"border-box",zIndex:L},l),onMouseEnter:M,onMouseLeave:D,onPointerEnter:j,onClick:x},k&&h.createElement(E,{prefixCls:i,arrow:k,arrowPos:C,align:A}),h.createElement(w,{cache:!p&&!g},q))})}))}),O=h.forwardRef(function(e,t){var n=e.children,r=e.getTriggerDOMNode,o=(0,y.Yr)(n),a=h.useCallback(function(e){(0,y.mH)(t,r?r(e):e)},[r]),i=(0,y.x1)(a,n.ref);return o?h.cloneElement(n,{ref:i}):n}),k=h.createContext(null);function C(e){return e?Array.isArray(e)?e:[e]:[]}var A=n(73193);function T(e,t,n,r){return t||(n?{motionName:"".concat(e,"-").concat(n)}:r?{motionName:r}:null)}function I(e){return e.ownerDocument.defaultView}function N(e){for(var t=[],n=null==e?void 0:e.parentElement,r=["hidden","scroll","clip","auto"];n;){var o=I(n).getComputedStyle(n);[o.overflowX,o.overflowY,o.overflow].some(function(e){return r.includes(e)})&&t.push(n),n=n.parentElement}return t}function R(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1;return Number.isNaN(e)?t:e}function _(e){return R(parseFloat(e),0)}function P(e,t){var n=(0,r.Z)({},e);return(t||[]).forEach(function(e){if(!(e instanceof HTMLBodyElement||e instanceof HTMLHtmlElement)){var t=I(e).getComputedStyle(e),r=t.overflow,o=t.overflowClipMargin,a=t.borderTopWidth,i=t.borderBottomWidth,l=t.borderLeftWidth,s=t.borderRightWidth,c=e.getBoundingClientRect(),u=e.offsetHeight,d=e.clientHeight,p=e.offsetWidth,f=e.clientWidth,m=_(a),g=_(i),h=_(l),b=_(s),v=R(Math.round(c.width/p*1e3)/1e3),y=R(Math.round(c.height/u*1e3)/1e3),E=m*y,S=h*v,w=0,x=0;if("clip"===r){var O=_(o);w=O*v,x=O*y}var k=c.x+S-w,C=c.y+E-x,A=k+c.width+2*w-S-b*v-(p-f-h-b)*v,T=C+c.height+2*x-E-g*y-(u-d-m-g)*y;n.left=Math.max(n.left,k),n.top=Math.max(n.top,C),n.right=Math.min(n.right,A),n.bottom=Math.min(n.bottom,T)}}),n}function L(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0,n="".concat(t),r=n.match(/^(.*)\%$/);return r?parseFloat(r[1])/100*e:parseFloat(n)}function M(e,t){var n=(0,o.Z)(t||[],2),r=n[0],a=n[1];return[L(e.width,r),L(e.height,a)]}function D(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"";return[e[0],e[1]]}function j(e,t){var n,r=t[0],o=t[1];return n="t"===r?e.y:"b"===r?e.y+e.height:e.y+e.height/2,{x:"l"===o?e.x:"r"===o?e.x+e.width:e.x+e.width/2,y:n}}function F(e,t){var n={t:"b",b:"t",l:"r",r:"l"};return e.map(function(e,r){return r===t?n[e]||"c":e}).join("")}var B=n(63787);n(53850);var Z=n(19223),U=["prefixCls","children","action","showAction","hideAction","popupVisible","defaultPopupVisible","onPopupVisibleChange","afterPopupVisibleChange","mouseEnterDelay","mouseLeaveDelay","focusDelay","blurDelay","mask","maskClosable","getPopupContainer","forceRender","autoDestroy","destroyPopupOnHide","popup","popupClassName","popupStyle","popupPlacement","builtinPlacements","popupAlign","zIndex","stretch","getPopupClassNameFromAlign","fresh","alignPoint","onPopupClick","onPopupAlign","arrow","popupMotion","maskMotion","popupTransitionName","popupAnimation","maskTransitionName","maskAnimation","className","getTriggerDOMNode"],z=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:i.Z;return h.forwardRef(function(t,n){var i,l,b,v,y,E,S,w,_,L,z,H,G,$,W,V,q,Y=t.prefixCls,K=void 0===Y?"rc-trigger-popup":Y,X=t.children,Q=t.action,J=t.showAction,ee=t.hideAction,et=t.popupVisible,en=t.defaultPopupVisible,er=t.onPopupVisibleChange,eo=t.afterPopupVisibleChange,ea=t.mouseEnterDelay,ei=t.mouseLeaveDelay,el=void 0===ei?.1:ei,es=t.focusDelay,ec=t.blurDelay,eu=t.mask,ed=t.maskClosable,ep=t.getPopupContainer,ef=t.forceRender,em=t.autoDestroy,eg=t.destroyPopupOnHide,eh=t.popup,eb=t.popupClassName,ev=t.popupStyle,ey=t.popupPlacement,eE=t.builtinPlacements,eS=void 0===eE?{}:eE,ew=t.popupAlign,ex=t.zIndex,eO=t.stretch,ek=t.getPopupClassNameFromAlign,eC=t.fresh,eA=t.alignPoint,eT=t.onPopupClick,eI=t.onPopupAlign,eN=t.arrow,eR=t.popupMotion,e_=t.maskMotion,eP=t.popupTransitionName,eL=t.popupAnimation,eM=t.maskTransitionName,eD=t.maskAnimation,ej=t.className,eF=t.getTriggerDOMNode,eB=(0,a.Z)(t,U),eZ=h.useState(!1),eU=(0,o.Z)(eZ,2),ez=eU[0],eH=eU[1];(0,m.Z)(function(){eH((0,g.Z)())},[]);var eG=h.useRef({}),e$=h.useContext(k),eW=h.useMemo(function(){return{registerSubPopup:function(e,t){eG.current[e]=t,null==e$||e$.registerSubPopup(e,t)}}},[e$]),eV=(0,f.Z)(),eq=h.useState(null),eY=(0,o.Z)(eq,2),eK=eY[0],eX=eY[1],eQ=(0,p.Z)(function(e){(0,u.S)(e)&&eK!==e&&eX(e),null==e$||e$.registerSubPopup(eV,e)}),eJ=h.useState(null),e0=(0,o.Z)(eJ,2),e1=e0[0],e2=e0[1],e4=h.useRef(null),e6=(0,p.Z)(function(e){(0,u.S)(e)&&e1!==e&&(e2(e),e4.current=e)}),e3=h.Children.only(X),e5=(null==e3?void 0:e3.props)||{},e8={},e9=(0,p.Z)(function(e){var t,n;return(null==e1?void 0:e1.contains(e))||(null===(t=(0,d.A)(e1))||void 0===t?void 0:t.host)===e||e===e1||(null==eK?void 0:eK.contains(e))||(null===(n=(0,d.A)(eK))||void 0===n?void 0:n.host)===e||e===eK||Object.values(eG.current).some(function(t){return(null==t?void 0:t.contains(e))||e===t})}),e7=T(K,eR,eL,eP),te=T(K,e_,eD,eM),tt=h.useState(en||!1),tn=(0,o.Z)(tt,2),tr=tn[0],to=tn[1],ta=null!=et?et:tr,ti=(0,p.Z)(function(e){void 0===et&&to(e)});(0,m.Z)(function(){to(et||!1)},[et]);var tl=h.useRef(ta);tl.current=ta;var ts=h.useRef([]);ts.current=[];var tc=(0,p.Z)(function(e){var t;ti(e),(null!==(t=ts.current[ts.current.length-1])&&void 0!==t?t:ta)!==e&&(ts.current.push(e),null==er||er(e))}),tu=h.useRef(),td=function(){clearTimeout(tu.current)},tp=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0;td(),0===t?tc(e):tu.current=setTimeout(function(){tc(e)},1e3*t)};h.useEffect(function(){return td},[]);var tf=h.useState(!1),tm=(0,o.Z)(tf,2),tg=tm[0],th=tm[1];(0,m.Z)(function(e){(!e||ta)&&th(!0)},[ta]);var tb=h.useState(null),tv=(0,o.Z)(tb,2),ty=tv[0],tE=tv[1],tS=h.useState([0,0]),tw=(0,o.Z)(tS,2),tx=tw[0],tO=tw[1],tk=function(e){tO([e.clientX,e.clientY])},tC=(i=eA?tx:e1,l=h.useState({ready:!1,offsetX:0,offsetY:0,offsetR:0,offsetB:0,arrowX:0,arrowY:0,scaleX:1,scaleY:1,align:eS[ey]||{}}),v=(b=(0,o.Z)(l,2))[0],y=b[1],E=h.useRef(0),S=h.useMemo(function(){return eK?N(eK):[]},[eK]),w=h.useRef({}),ta||(w.current={}),_=(0,p.Z)(function(){if(eK&&i&&ta){var e,t,n,a,l,s,c,d=eK.ownerDocument,p=I(eK).getComputedStyle(eK),f=p.width,m=p.height,g=p.position,h=eK.style.left,b=eK.style.top,v=eK.style.right,E=eK.style.bottom,x=eK.style.overflow,O=(0,r.Z)((0,r.Z)({},eS[ey]),ew),k=d.createElement("div");if(null===(e=eK.parentElement)||void 0===e||e.appendChild(k),k.style.left="".concat(eK.offsetLeft,"px"),k.style.top="".concat(eK.offsetTop,"px"),k.style.position=g,k.style.height="".concat(eK.offsetHeight,"px"),k.style.width="".concat(eK.offsetWidth,"px"),eK.style.left="0",eK.style.top="0",eK.style.right="auto",eK.style.bottom="auto",eK.style.overflow="hidden",Array.isArray(i))n={x:i[0],y:i[1],width:0,height:0};else{var C=i.getBoundingClientRect();n={x:C.x,y:C.y,width:C.width,height:C.height}}var T=eK.getBoundingClientRect(),N=d.documentElement,_=N.clientWidth,L=N.clientHeight,B=N.scrollWidth,Z=N.scrollHeight,U=N.scrollTop,z=N.scrollLeft,H=T.height,G=T.width,$=n.height,W=n.width,V=O.htmlRegion,q="visible",Y="visibleFirst";"scroll"!==V&&V!==Y&&(V=q);var K=V===Y,X=P({left:-z,top:-U,right:B-z,bottom:Z-U},S),Q=P({left:0,top:0,right:_,bottom:L},S),J=V===q?Q:X,ee=K?Q:J;eK.style.left="auto",eK.style.top="auto",eK.style.right="0",eK.style.bottom="0";var et=eK.getBoundingClientRect();eK.style.left=h,eK.style.top=b,eK.style.right=v,eK.style.bottom=E,eK.style.overflow=x,null===(t=eK.parentElement)||void 0===t||t.removeChild(k);var en=R(Math.round(G/parseFloat(f)*1e3)/1e3),er=R(Math.round(H/parseFloat(m)*1e3)/1e3);if(!(0===en||0===er||(0,u.S)(i)&&!(0,A.Z)(i))){var eo=O.offset,ea=O.targetOffset,ei=M(T,eo),el=(0,o.Z)(ei,2),es=el[0],ec=el[1],eu=M(n,ea),ed=(0,o.Z)(eu,2),ep=ed[0],ef=ed[1];n.x-=ep,n.y-=ef;var em=O.points||[],eg=(0,o.Z)(em,2),eh=eg[0],eb=D(eg[1]),ev=D(eh),eE=j(n,eb),ex=j(T,ev),eO=(0,r.Z)({},O),ek=eE.x-ex.x+es,eC=eE.y-ex.y+ec,eA=tt(ek,eC),eT=tt(ek,eC,Q),eN=j(n,["t","l"]),eR=j(T,["t","l"]),e_=j(n,["b","r"]),eP=j(T,["b","r"]),eL=O.overflow||{},eM=eL.adjustX,eD=eL.adjustY,ej=eL.shiftX,eF=eL.shiftY,eB=function(e){return"boolean"==typeof e?e:e>=0};tn();var eZ=eB(eD),eU=ev[0]===eb[0];if(eZ&&"t"===ev[0]&&(l>ee.bottom||w.current.bt)){var ez=eC;eU?ez-=H-$:ez=eN.y-eP.y-ec;var eH=tt(ek,ez),eG=tt(ek,ez,Q);eH>eA||eH===eA&&(!K||eG>=eT)?(w.current.bt=!0,eC=ez,ec=-ec,eO.points=[F(ev,0),F(eb,0)]):w.current.bt=!1}if(eZ&&"b"===ev[0]&&(aeA||eW===eA&&(!K||eV>=eT)?(w.current.tb=!0,eC=e$,ec=-ec,eO.points=[F(ev,0),F(eb,0)]):w.current.tb=!1}var eq=eB(eM),eY=ev[1]===eb[1];if(eq&&"l"===ev[1]&&(c>ee.right||w.current.rl)){var eX=ek;eY?eX-=G-W:eX=eN.x-eP.x-es;var eQ=tt(eX,eC),eJ=tt(eX,eC,Q);eQ>eA||eQ===eA&&(!K||eJ>=eT)?(w.current.rl=!0,ek=eX,es=-es,eO.points=[F(ev,1),F(eb,1)]):w.current.rl=!1}if(eq&&"r"===ev[1]&&(seA||e1===eA&&(!K||e2>=eT)?(w.current.lr=!0,ek=e0,es=-es,eO.points=[F(ev,1),F(eb,1)]):w.current.lr=!1}tn();var e4=!0===ej?0:ej;"number"==typeof e4&&(sQ.right&&(ek-=c-Q.right-es,n.x>Q.right-e4&&(ek+=n.x-Q.right+e4)));var e6=!0===eF?0:eF;"number"==typeof e6&&(aQ.bottom&&(eC-=l-Q.bottom-ec,n.y>Q.bottom-e6&&(eC+=n.y-Q.bottom+e6)));var e3=T.x+ek,e5=T.y+eC,e8=n.x,e9=n.y;null==eI||eI(eK,eO);var e7=et.right-T.x-(ek+T.width),te=et.bottom-T.y-(eC+T.height);y({ready:!0,offsetX:ek/en,offsetY:eC/er,offsetR:e7/en,offsetB:te/er,arrowX:((Math.max(e3,e8)+Math.min(e3+G,e8+W))/2-e3)/en,arrowY:((Math.max(e5,e9)+Math.min(e5+H,e9+$))/2-e5)/er,scaleX:en,scaleY:er,align:eO})}function tt(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:J,r=T.x+e,o=T.y+t,a=Math.max(r,n.left),i=Math.max(o,n.top);return Math.max(0,(Math.min(r+G,n.right)-a)*(Math.min(o+H,n.bottom)-i))}function tn(){l=(a=T.y+eC)+H,c=(s=T.x+ek)+G}}}),L=function(){y(function(e){return(0,r.Z)((0,r.Z)({},e),{},{ready:!1})})},(0,m.Z)(L,[ey]),(0,m.Z)(function(){ta||L()},[ta]),[v.ready,v.offsetX,v.offsetY,v.offsetR,v.offsetB,v.arrowX,v.arrowY,v.scaleX,v.scaleY,v.align,function(){E.current+=1;var e=E.current;Promise.resolve().then(function(){E.current===e&&_()})}]),tA=(0,o.Z)(tC,11),tT=tA[0],tI=tA[1],tN=tA[2],tR=tA[3],t_=tA[4],tP=tA[5],tL=tA[6],tM=tA[7],tD=tA[8],tj=tA[9],tF=tA[10],tB=(z=void 0===Q?"hover":Q,h.useMemo(function(){var e=C(null!=J?J:z),t=C(null!=ee?ee:z),n=new Set(e),r=new Set(t);return ez&&(n.has("hover")&&(n.delete("hover"),n.add("click")),r.has("hover")&&(r.delete("hover"),r.add("click"))),[n,r]},[ez,z,J,ee])),tZ=(0,o.Z)(tB,2),tU=tZ[0],tz=tZ[1],tH=tU.has("click"),tG=tz.has("click")||tz.has("contextMenu"),t$=(0,p.Z)(function(){tg||tF()});H=function(){tl.current&&eA&&tG&&tp(!1)},(0,m.Z)(function(){if(ta&&e1&&eK){var e=N(e1),t=N(eK),n=I(eK),r=new Set([n].concat((0,B.Z)(e),(0,B.Z)(t)));function o(){t$(),H()}return r.forEach(function(e){e.addEventListener("scroll",o,{passive:!0})}),n.addEventListener("resize",o,{passive:!0}),t$(),function(){r.forEach(function(e){e.removeEventListener("scroll",o),n.removeEventListener("resize",o)})}}},[ta,e1,eK]),(0,m.Z)(function(){t$()},[tx,ey]),(0,m.Z)(function(){ta&&!(null!=eS&&eS[ey])&&t$()},[JSON.stringify(ew)]);var tW=h.useMemo(function(){var e=function(e,t,n,r){for(var o=n.points,a=Object.keys(e),i=0;i0&&void 0!==arguments[0]?arguments[0]:[],t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],n=arguments.length>2?arguments[2]:void 0;return n?e[0]===t[0]:e[0]===t[0]&&e[1]===t[1]}(null===(l=e[s])||void 0===l?void 0:l.points,o,r))return"".concat(t,"-placement-").concat(s)}return""}(eS,K,tj,eA);return s()(e,null==ek?void 0:ek(tj))},[tj,ek,eS,K,eA]);h.useImperativeHandle(n,function(){return{nativeElement:e4.current,forceAlign:t$}});var tV=h.useState(0),tq=(0,o.Z)(tV,2),tY=tq[0],tK=tq[1],tX=h.useState(0),tQ=(0,o.Z)(tX,2),tJ=tQ[0],t0=tQ[1],t1=function(){if(eO&&e1){var e=e1.getBoundingClientRect();tK(e.width),t0(e.height)}};function t2(e,t,n,r){e8[e]=function(o){var a;null==r||r(o),tp(t,n);for(var i=arguments.length,l=Array(i>1?i-1:0),s=1;s1?n-1:0),o=1;o1?n-1:0),o=1;o{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},t),o.createElement("path",{d:"M11.9999 13.1714L16.9497 8.22168L18.3639 9.63589L11.9999 15.9999L5.63599 9.63589L7.0502 8.22168L11.9999 13.1714Z"}))}},8903:function(e,t,n){n.d(t,{Z:function(){return a}});var r=n(69703),o=n(64090);let a=e=>{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},t),o.createElement("path",{d:"M12 22C6.47715 22 2 17.5228 2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12C22 17.5228 17.5228 22 12 22ZM12 10.5858L9.17157 7.75736L7.75736 9.17157L10.5858 12L7.75736 14.8284L9.17157 16.2426L12 13.4142L14.8284 16.2426L16.2426 14.8284L13.4142 12L16.2426 9.17157L14.8284 7.75736L12 10.5858Z"}))}},57750:function(e,t,n){n.d(t,{Z:function(){return eg}});var r=n(69703),o=n(64090),a=n(26587),i=n(65558),l=n(75504),s=n(30638),c=n(80509),u=n.n(c),d=n(5037),p=n.n(d),f=n(71292),m=n.n(f),g=n(96240),h=n.n(g),b=n(93574),v=n.n(b),y=n(72996),E=n(84487),S=n(7986),w=n(71594),x=n(68139),O=n(20757),k=n(9586),C=n(765),A=["layout","type","stroke","connectNulls","isRange","ref"];function T(e){return(T="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e})(e)}function I(){return(I=Object.assign?Object.assign.bind():function(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0)&&Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}(a,A));return o.createElement(S.m,{clipPath:n?"url(#clipPath-".concat(r,")"):null},o.createElement(y.H,I({},(0,C.L6)(d,!0),{points:e,connectNulls:c,type:l,baseLine:t,layout:i,stroke:"none",className:"recharts-area-area"})),"none"!==s&&o.createElement(y.H,I({},(0,C.L6)(this.props,!1),{className:"recharts-area-curve",layout:i,type:l,connectNulls:c,fill:"none",points:e})),"none"!==s&&u&&o.createElement(y.H,I({},(0,C.L6)(this.props,!1),{className:"recharts-area-curve",layout:i,type:l,connectNulls:c,fill:"none",points:t})))}},{key:"renderAreaWithAnimation",value:function(e,t){var n=this,r=this.props,a=r.points,i=r.baseLine,l=r.isAnimationActive,c=r.animationBegin,u=r.animationDuration,d=r.animationEasing,p=r.animationId,f=this.state,g=f.prevPoints,b=f.prevBaseLine;return o.createElement(s.ZP,{begin:c,duration:u,isActive:l,easing:d,from:{t:0},to:{t:1},key:"area-".concat(p),onAnimationEnd:this.handleAnimationEnd,onAnimationStart:this.handleAnimationStart},function(r){var l=r.t;if(g){var s,c=g.length/a.length,u=a.map(function(e,t){var n=Math.floor(t*c);if(g[n]){var r=g[n],o=(0,O.k4)(r.x,e.x),a=(0,O.k4)(r.y,e.y);return R(R({},e),{},{x:o(l),y:a(l)})}return e});return s=(0,O.hj)(i)&&"number"==typeof i?(0,O.k4)(b,i)(l):m()(i)||h()(i)?(0,O.k4)(b,0)(l):i.map(function(e,t){var n=Math.floor(t*c);if(b[n]){var r=b[n],o=(0,O.k4)(r.x,e.x),a=(0,O.k4)(r.y,e.y);return R(R({},e),{},{x:o(l),y:a(l)})}return e}),n.renderAreaStatically(u,s,e,t)}return o.createElement(S.m,null,o.createElement("defs",null,o.createElement("clipPath",{id:"animationClipPath-".concat(t)},n.renderClipRect(l))),o.createElement(S.m,{clipPath:"url(#animationClipPath-".concat(t,")")},n.renderAreaStatically(a,i,e,t)))})}},{key:"renderArea",value:function(e,t){var n=this.props,r=n.points,o=n.baseLine,a=n.isAnimationActive,i=this.state,l=i.prevPoints,s=i.prevBaseLine,c=i.totalLength;return a&&r&&r.length&&(!l&&c>0||!v()(l,r)||!v()(s,o))?this.renderAreaWithAnimation(e,t):this.renderAreaStatically(r,o,e,t)}},{key:"render",value:function(){var e,t=this.props,n=t.hide,r=t.dot,a=t.points,i=t.className,s=t.top,c=t.left,u=t.xAxis,d=t.yAxis,p=t.width,f=t.height,g=t.isAnimationActive,h=t.id;if(n||!a||!a.length)return null;var b=this.state.isAnimationFinished,v=1===a.length,y=(0,l.Z)("recharts-area",i),E=u&&u.allowDataOverflow,x=d&&d.allowDataOverflow,O=E||x,k=m()(h)?this.id:h,A=null!==(e=(0,C.L6)(r,!1))&&void 0!==e?e:{r:3,strokeWidth:2},T=A.r,I=A.strokeWidth,N=((0,C.$k)(r)?r:{}).clipDot,R=void 0===N||N,_=2*(void 0===T?3:T)+(void 0===I?2:I);return o.createElement(S.m,{className:y},E||x?o.createElement("defs",null,o.createElement("clipPath",{id:"clipPath-".concat(k)},o.createElement("rect",{x:E?c:c-p/2,y:x?s:s-f/2,width:E?p:2*p,height:x?f:2*f})),!R&&o.createElement("clipPath",{id:"clipPath-dots-".concat(k)},o.createElement("rect",{x:c-_/2,y:s-_/2,width:p+_,height:f+_}))):null,v?null:this.renderArea(O,k),(r||v)&&this.renderDots(O,R,k),(!g||b)&&w.e.renderCallByParent(this.props,a))}}],r=[{key:"getDerivedStateFromProps",value:function(e,t){return e.animationId!==t.prevAnimationId?{prevAnimationId:e.animationId,curPoints:e.points,curBaseLine:e.baseLine,prevPoints:t.curPoints,prevBaseLine:t.curBaseLine}:e.points!==t.curPoints||e.baseLine!==t.curBaseLine?{curPoints:e.points,curBaseLine:e.baseLine}:null}}],n&&_(i.prototype,n),r&&_(i,r),Object.defineProperty(i,"prototype",{writable:!1}),i}(o.PureComponent);D(F,"displayName","Area"),D(F,"defaultProps",{stroke:"#3182bd",fill:"#3182bd",fillOpacity:.6,xAxisId:0,yAxisId:0,legendType:"line",connectNulls:!1,points:[],dot:!1,activeDot:!0,hide:!1,isAnimationActive:!x.x.isSsr,animationBegin:0,animationDuration:1500,animationEasing:"ease"}),D(F,"getBaseValue",function(e,t,n,r){var o=e.layout,a=e.baseValue,i=t.props.baseValue,l=null!=i?i:a;if((0,O.hj)(l)&&"number"==typeof l)return l;var s="horizontal"===o?r:n,c=s.scale.domain();if("number"===s.type){var u=Math.max(c[0],c[1]),d=Math.min(c[0],c[1]);return"dataMin"===l?d:"dataMax"===l?u:u<0?u:Math.max(Math.min(c[0],c[1]),0)}return"dataMin"===l?c[0]:"dataMax"===l?c[1]:c[0]}),D(F,"getComposedData",function(e){var t,n=e.props,r=e.item,o=e.xAxis,a=e.yAxis,i=e.xAxisTicks,l=e.yAxisTicks,s=e.bandSize,c=e.dataKey,u=e.stackedData,d=e.dataStartIndex,p=e.displayedData,f=e.offset,m=n.layout,g=u&&u.length,h=F.getBaseValue(n,r,o,a),b="horizontal"===m,v=!1,y=p.map(function(e,t){g?n=u[d+t]:Array.isArray(n=(0,k.F$)(e,c))?v=!0:n=[h,n];var n,r=null==n[1]||g&&null==(0,k.F$)(e,c);return b?{x:(0,k.Hv)({axis:o,ticks:i,bandSize:s,entry:e,index:t}),y:r?null:a.scale(n[1]),value:n,payload:e}:{x:r?null:o.scale(n[1]),y:(0,k.Hv)({axis:a,ticks:l,bandSize:s,entry:e,index:t}),value:n,payload:e}});return t=g||v?y.map(function(e){var t=Array.isArray(e.value)?e.value[0]:null;return b?{x:e.x,y:null!=t&&null!=e.y?a.scale(t):null}:{x:null!=t?o.scale(t):null,y:e.y}}):b?a.scale(h):o.scale(h),R({points:y,baseLine:t,layout:m,isRange:v},f)}),D(F,"renderDotItem",function(e,t){return o.isValidElement(e)?o.cloneElement(e,t):u()(e)?e(t):o.createElement(E.o,I({},t,{className:"recharts-area-dot"}))});var B=n(23356),Z=n(22983),U=n(12627),z=(0,i.z)({chartName:"AreaChart",GraphicalChild:F,axisComponents:[{axisType:"xAxis",AxisComp:B.K},{axisType:"yAxis",AxisComp:Z.B}],formatAxisMap:U.t9}),H=n(38333),G=n(10166),$=n(94866),W=n(99355),V=["type","layout","connectNulls","ref"];function q(e){return(q="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e})(e)}function Y(){return(Y=Object.assign?Object.assign.bind():function(e){for(var t=1;te.length)&&(t=e.length);for(var n=0,r=Array(t);na){s=[].concat(Q(r.slice(0,c)),[a-u]);break}var d=s.length%2==0?[0,l]:[l];return[].concat(Q(i.repeat(r,Math.floor(t/o))),Q(s),d).map(function(e){return"".concat(e,"px")}).join(", ")}),eo(en(e),"id",(0,O.EL)("recharts-line-")),eo(en(e),"pathRef",function(t){e.mainCurve=t}),eo(en(e),"handleAnimationEnd",function(){e.setState({isAnimationFinished:!0}),e.props.onAnimationEnd&&e.props.onAnimationEnd()}),eo(en(e),"handleAnimationStart",function(){e.setState({isAnimationFinished:!1}),e.props.onAnimationStart&&e.props.onAnimationStart()}),e}return n=[{key:"componentDidMount",value:function(){if(this.props.isAnimationActive){var e=this.getTotalLength();this.setState({totalLength:e})}}},{key:"componentDidUpdate",value:function(){if(this.props.isAnimationActive){var e=this.getTotalLength();e!==this.state.totalLength&&this.setState({totalLength:e})}}},{key:"getTotalLength",value:function(){var e=this.mainCurve;try{return e&&e.getTotalLength&&e.getTotalLength()||0}catch(e){return 0}}},{key:"renderErrorBar",value:function(e,t){if(this.props.isAnimationActive&&!this.state.isAnimationFinished)return null;var n=this.props,r=n.points,a=n.xAxis,i=n.yAxis,l=n.layout,s=n.children,c=(0,C.NN)(s,W.W);if(!c)return null;var u=function(e,t){return{x:e.x,y:e.y,value:e.value,errorVal:(0,k.F$)(e.payload,t)}};return o.createElement(S.m,{clipPath:e?"url(#clipPath-".concat(t,")"):null},c.map(function(e){return o.cloneElement(e,{key:"bar-".concat(e.props.dataKey),data:r,xAxis:a,yAxis:i,layout:l,dataPointFormatter:u})}))}},{key:"renderDots",value:function(e,t,n){if(this.props.isAnimationActive&&!this.state.isAnimationFinished)return null;var r=this.props,a=r.dot,l=r.points,s=r.dataKey,c=(0,C.L6)(this.props,!1),u=(0,C.L6)(a,!0),d=l.map(function(e,t){var n=X(X(X({key:"dot-".concat(t),r:3},c),u),{},{value:e.value,dataKey:s,cx:e.x,cy:e.y,index:t,payload:e.payload});return i.renderDotItem(a,n)}),p={clipPath:e?"url(#clipPath-".concat(t?"":"dots-").concat(n,")"):null};return o.createElement(S.m,Y({className:"recharts-line-dots",key:"dots"},p),d)}},{key:"renderCurveStatically",value:function(e,t,n,r){var a=this.props,i=a.type,l=a.layout,s=a.connectNulls,c=(a.ref,function(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0)&&Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}(a,V)),u=X(X(X({},(0,C.L6)(c,!0)),{},{fill:"none",className:"recharts-line-curve",clipPath:t?"url(#clipPath-".concat(n,")"):null,points:e},r),{},{type:i,layout:l,connectNulls:s});return o.createElement(y.H,Y({},u,{pathRef:this.pathRef}))}},{key:"renderCurveWithAnimation",value:function(e,t){var n=this,r=this.props,a=r.points,i=r.strokeDasharray,l=r.isAnimationActive,c=r.animationBegin,u=r.animationDuration,d=r.animationEasing,p=r.animationId,f=r.animateNewValues,m=r.width,g=r.height,h=this.state,b=h.prevPoints,v=h.totalLength;return o.createElement(s.ZP,{begin:c,duration:u,isActive:l,easing:d,from:{t:0},to:{t:1},key:"line-".concat(p),onAnimationEnd:this.handleAnimationEnd,onAnimationStart:this.handleAnimationStart},function(r){var o,l=r.t;if(b){var s=b.length/a.length,c=a.map(function(e,t){var n=Math.floor(t*s);if(b[n]){var r=b[n],o=(0,O.k4)(r.x,e.x),a=(0,O.k4)(r.y,e.y);return X(X({},e),{},{x:o(l),y:a(l)})}if(f){var i=(0,O.k4)(2*m,e.x),c=(0,O.k4)(g/2,e.y);return X(X({},e),{},{x:i(l),y:c(l)})}return X(X({},e),{},{x:e.x,y:e.y})});return n.renderCurveStatically(c,e,t)}var u=(0,O.k4)(0,v)(l);if(i){var d="".concat(i).split(/[,\s]+/gim).map(function(e){return parseFloat(e)});o=n.getStrokeDasharray(u,v,d)}else o=n.generateSimpleStrokeDasharray(v,u);return n.renderCurveStatically(a,e,t,{strokeDasharray:o})})}},{key:"renderCurve",value:function(e,t){var n=this.props,r=n.points,o=n.isAnimationActive,a=this.state,i=a.prevPoints,l=a.totalLength;return o&&r&&r.length&&(!i&&l>0||!v()(i,r))?this.renderCurveWithAnimation(e,t):this.renderCurveStatically(r,e,t)}},{key:"render",value:function(){var e,t=this.props,n=t.hide,r=t.dot,a=t.points,i=t.className,s=t.xAxis,c=t.yAxis,u=t.top,d=t.left,p=t.width,f=t.height,g=t.isAnimationActive,h=t.id;if(n||!a||!a.length)return null;var b=this.state.isAnimationFinished,v=1===a.length,y=(0,l.Z)("recharts-line",i),E=s&&s.allowDataOverflow,x=c&&c.allowDataOverflow,O=E||x,k=m()(h)?this.id:h,A=null!==(e=(0,C.L6)(r,!1))&&void 0!==e?e:{r:3,strokeWidth:2},T=A.r,I=A.strokeWidth,N=((0,C.$k)(r)?r:{}).clipDot,R=void 0===N||N,_=2*(void 0===T?3:T)+(void 0===I?2:I);return o.createElement(S.m,{className:y},E||x?o.createElement("defs",null,o.createElement("clipPath",{id:"clipPath-".concat(k)},o.createElement("rect",{x:E?d:d-p/2,y:x?u:u-f/2,width:E?p:2*p,height:x?f:2*f})),!R&&o.createElement("clipPath",{id:"clipPath-dots-".concat(k)},o.createElement("rect",{x:d-_/2,y:u-_/2,width:p+_,height:f+_}))):null,!v&&this.renderCurve(O,k),this.renderErrorBar(O,k),(v||r)&&this.renderDots(O,R,k),(!g||b)&&w.e.renderCallByParent(this.props,a))}}],r=[{key:"getDerivedStateFromProps",value:function(e,t){return e.animationId!==t.prevAnimationId?{prevAnimationId:e.animationId,curPoints:e.points,prevPoints:t.curPoints}:e.points!==t.curPoints?{curPoints:e.points}:null}},{key:"repeat",value:function(e,t){for(var n=e.length%2!=0?[].concat(Q(e),[0]):e,r=[],o=0;o{let{data:n=[],categories:i=[],index:l,stack:s=!1,colors:c=ep.s,valueFormatter:u=em.Cj,startEndOnly:d=!1,showXAxis:p=!0,showYAxis:f=!0,yAxisWidth:m=56,intervalType:g="equidistantPreserveStart",showAnimation:h=!1,animationDuration:b=900,showTooltip:v=!0,showLegend:y=!0,showGridLines:S=!0,showGradient:w=!0,autoMinValue:x=!1,curveType:O="linear",minValue:k,maxValue:C,connectNulls:A=!1,allowDecimals:T=!0,noDataText:I,className:N,onValueChange:R,enableLegendSlider:_=!1,customTooltip:P,rotateLabelX:L,tickGap:M=5}=e,D=(0,r._T)(e,["data","categories","index","stack","colors","valueFormatter","startEndOnly","showXAxis","showYAxis","yAxisWidth","intervalType","showAnimation","animationDuration","showTooltip","showLegend","showGridLines","showGradient","autoMinValue","curveType","minValue","maxValue","connectNulls","allowDecimals","noDataText","className","onValueChange","enableLegendSlider","customTooltip","rotateLabelX","tickGap"]),j=(p||f)&&(!d||f)?20:0,[U,W]=(0,o.useState)(60),[V,q]=(0,o.useState)(void 0),[Y,K]=(0,o.useState)(void 0),X=(0,eu.me)(i,c),Q=(0,eu.i4)(x,k,C),J=!!R;function ee(e){J&&(e===Y&&!V||(0,eu.FB)(n,e)&&V&&V.dataKey===e?(K(void 0),null==R||R(null)):(K(e),null==R||R({eventType:"category",categoryClicked:e})),q(void 0))}return o.createElement("div",Object.assign({ref:t,className:(0,ef.q)("w-full h-80",N)},D),o.createElement(a.h,{className:"h-full w-full"},(null==n?void 0:n.length)?o.createElement(z,{data:n,onClick:J&&(Y||V)?()=>{q(void 0),K(void 0),null==R||R(null)}:void 0},S?o.createElement(H.q,{className:(0,ef.q)("stroke-1","stroke-tremor-border","dark:stroke-dark-tremor-border"),horizontal:!0,vertical:!1}):null,o.createElement(B.K,{padding:{left:j,right:j},hide:!p,dataKey:l,tick:{transform:"translate(0, 6)"},ticks:d?[n[0][l],n[n.length-1][l]]:void 0,fill:"",stroke:"",className:(0,ef.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),interval:d?"preserveStartEnd":g,tickLine:!1,axisLine:!1,minTickGap:M,angle:null==L?void 0:L.angle,dy:null==L?void 0:L.verticalShift,height:null==L?void 0:L.xAxisHeight}),o.createElement(Z.B,{width:m,hide:!f,axisLine:!1,tickLine:!1,type:"number",domain:Q,tick:{transform:"translate(-3, 0)"},fill:"",stroke:"",className:(0,ef.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickFormatter:u,allowDecimals:T}),o.createElement(G.u,{wrapperStyle:{outline:"none"},isAnimationActive:!1,cursor:{stroke:"#d1d5db",strokeWidth:1},content:v?e=>{let{active:t,payload:n,label:r}=e;return P?o.createElement(P,{payload:null==n?void 0:n.map(e=>{var t;return Object.assign(Object.assign({},e),{color:null!==(t=X.get(e.dataKey))&&void 0!==t?t:ed.fr.Gray})}),active:t,label:r}):o.createElement(es.ZP,{active:t,payload:n,label:r,valueFormatter:u,categoryColors:X})}:o.createElement(o.Fragment,null),position:{y:0}}),y?o.createElement($.D,{verticalAlign:"top",height:U,content:e=>{let{payload:t}=e;return(0,el.Z)({payload:t},X,W,Y,J?e=>ee(e):void 0,_)}}):null,i.map(e=>{var t,n;return o.createElement("defs",{key:e},w?o.createElement("linearGradient",{className:(0,em.bM)(null!==(t=X.get(e))&&void 0!==t?t:ed.fr.Gray,ep.K.text).textColor,id:X.get(e),x1:"0",y1:"0",x2:"0",y2:"1"},o.createElement("stop",{offset:"5%",stopColor:"currentColor",stopOpacity:V||Y&&Y!==e?.15:.4}),o.createElement("stop",{offset:"95%",stopColor:"currentColor",stopOpacity:0})):o.createElement("linearGradient",{className:(0,em.bM)(null!==(n=X.get(e))&&void 0!==n?n:ed.fr.Gray,ep.K.text).textColor,id:X.get(e),x1:"0",y1:"0",x2:"0",y2:"1"},o.createElement("stop",{stopColor:"currentColor",stopOpacity:V||Y&&Y!==e?.1:.3})))}),i.map(e=>{var t;return o.createElement(F,{className:(0,em.bM)(null!==(t=X.get(e))&&void 0!==t?t:ed.fr.Gray,ep.K.text).strokeColor,strokeOpacity:V||Y&&Y!==e?.3:1,activeDot:e=>{var t;let{cx:r,cy:a,stroke:i,strokeLinecap:l,strokeLinejoin:s,strokeWidth:c,dataKey:u}=e;return o.createElement(E.o,{className:(0,ef.q)("stroke-tremor-background dark:stroke-dark-tremor-background",R?"cursor-pointer":"",(0,em.bM)(null!==(t=X.get(u))&&void 0!==t?t:ed.fr.Gray,ep.K.text).fillColor),cx:r,cy:a,r:5,fill:"",stroke:i,strokeLinecap:l,strokeLinejoin:s,strokeWidth:c,onClick:(t,r)=>{r.stopPropagation(),J&&(e.index===(null==V?void 0:V.index)&&e.dataKey===(null==V?void 0:V.dataKey)||(0,eu.FB)(n,e.dataKey)&&Y&&Y===e.dataKey?(K(void 0),q(void 0),null==R||R(null)):(K(e.dataKey),q({index:e.index,dataKey:e.dataKey}),null==R||R(Object.assign({eventType:"dot",categoryClicked:e.dataKey},e.payload))))}})},dot:t=>{var r;let{stroke:a,strokeLinecap:i,strokeLinejoin:l,strokeWidth:s,cx:c,cy:u,dataKey:d,index:p}=t;return(0,eu.FB)(n,e)&&!(V||Y&&Y!==e)||(null==V?void 0:V.index)===p&&(null==V?void 0:V.dataKey)===e?o.createElement(E.o,{key:p,cx:c,cy:u,r:5,stroke:a,fill:"",strokeLinecap:i,strokeLinejoin:l,strokeWidth:s,className:(0,ef.q)("stroke-tremor-background dark:stroke-dark-tremor-background",R?"cursor-pointer":"",(0,em.bM)(null!==(r=X.get(d))&&void 0!==r?r:ed.fr.Gray,ep.K.text).fillColor)}):o.createElement(o.Fragment,{key:p})},key:e,name:e,type:O,dataKey:e,stroke:"",fill:"url(#".concat(X.get(e),")"),strokeWidth:2,strokeLinejoin:"round",strokeLinecap:"round",isAnimationActive:h,animationDuration:b,stackId:s?"a":void 0,connectNulls:A})}),R?i.map(e=>o.createElement(ei,{className:(0,ef.q)("cursor-pointer"),strokeOpacity:0,key:e,name:e,type:O,dataKey:e,stroke:"transparent",fill:"transparent",legendType:"none",tooltipType:"none",strokeWidth:12,connectNulls:A,onClick:(e,t)=>{t.stopPropagation();let{name:n}=e;ee(n)}})):null):o.createElement(ec.Z,{noDataText:I})))});eg.displayName="AreaChart"},44041:function(e,t,n){n.d(t,{Z:function(){return x}});var r=n(69703),o=n(54942),a=n(2898),i=n(99250),l=n(65492),s=n(64090),c=n(26587),u=n(65558),d=n(28485),p=n(23356),f=n(22983),m=n(12627),g=(0,u.z)({chartName:"BarChart",GraphicalChild:d.$,defaultTooltipEventType:"axis",validateTooltipEventTypes:["axis","item"],axisComponents:[{axisType:"xAxis",AxisComp:p.K},{axisType:"yAxis",AxisComp:f.B}],formatAxisMap:m.t9}),h=n(38333),b=n(10166),v=n(94866),y=n(17280),E=n(30470),S=n(77448),w=n(36342);let x=s.forwardRef((e,t)=>{let{data:n=[],categories:u=[],index:m,colors:x=a.s,valueFormatter:O=l.Cj,layout:k="horizontal",stack:C=!1,relative:A=!1,startEndOnly:T=!1,animationDuration:I=900,showAnimation:N=!1,showXAxis:R=!0,showYAxis:_=!0,yAxisWidth:P=56,intervalType:L="equidistantPreserveStart",showTooltip:M=!0,showLegend:D=!0,showGridLines:j=!0,autoMinValue:F=!1,minValue:B,maxValue:Z,allowDecimals:U=!0,noDataText:z,onValueChange:H,enableLegendSlider:G=!1,customTooltip:$,rotateLabelX:W,tickGap:V=5,className:q}=e,Y=(0,r._T)(e,["data","categories","index","colors","valueFormatter","layout","stack","relative","startEndOnly","animationDuration","showAnimation","showXAxis","showYAxis","yAxisWidth","intervalType","showTooltip","showLegend","showGridLines","autoMinValue","minValue","maxValue","allowDecimals","noDataText","onValueChange","enableLegendSlider","customTooltip","rotateLabelX","tickGap","className"]),K=R||_?20:0,[X,Q]=(0,s.useState)(60),J=(0,w.me)(u,x),[ee,et]=s.useState(void 0),[en,er]=(0,s.useState)(void 0),eo=!!H;function ea(e,t,n){var r,o,a,i;n.stopPropagation(),H&&((0,w.vZ)(ee,Object.assign(Object.assign({},e.payload),{value:e.value}))?(er(void 0),et(void 0),null==H||H(null)):(er(null===(o=null===(r=e.tooltipPayload)||void 0===r?void 0:r[0])||void 0===o?void 0:o.dataKey),et(Object.assign(Object.assign({},e.payload),{value:e.value})),null==H||H(Object.assign({eventType:"bar",categoryClicked:null===(i=null===(a=e.tooltipPayload)||void 0===a?void 0:a[0])||void 0===i?void 0:i.dataKey},e.payload))))}let ei=(0,w.i4)(F,B,Z);return s.createElement("div",Object.assign({ref:t,className:(0,i.q)("w-full h-80",q)},Y),s.createElement(c.h,{className:"h-full w-full"},(null==n?void 0:n.length)?s.createElement(g,{data:n,stackOffset:C?"sign":A?"expand":"none",layout:"vertical"===k?"vertical":"horizontal",onClick:eo&&(en||ee)?()=>{et(void 0),er(void 0),null==H||H(null)}:void 0},j?s.createElement(h.q,{className:(0,i.q)("stroke-1","stroke-tremor-border","dark:stroke-dark-tremor-border"),horizontal:"vertical"!==k,vertical:"vertical"===k}):null,"vertical"!==k?s.createElement(p.K,{padding:{left:K,right:K},hide:!R,dataKey:m,interval:T?"preserveStartEnd":L,tick:{transform:"translate(0, 6)"},ticks:T?[n[0][m],n[n.length-1][m]]:void 0,fill:"",stroke:"",className:(0,i.q)("mt-4 text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickLine:!1,axisLine:!1,angle:null==W?void 0:W.angle,dy:null==W?void 0:W.verticalShift,height:null==W?void 0:W.xAxisHeight,minTickGap:V}):s.createElement(p.K,{hide:!R,type:"number",tick:{transform:"translate(-3, 0)"},domain:ei,fill:"",stroke:"",className:(0,i.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickLine:!1,axisLine:!1,tickFormatter:O,minTickGap:V,allowDecimals:U,angle:null==W?void 0:W.angle,dy:null==W?void 0:W.verticalShift,height:null==W?void 0:W.xAxisHeight}),"vertical"!==k?s.createElement(f.B,{width:P,hide:!_,axisLine:!1,tickLine:!1,type:"number",domain:ei,tick:{transform:"translate(-3, 0)"},fill:"",stroke:"",className:(0,i.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickFormatter:A?e=>"".concat((100*e).toString()," %"):O,allowDecimals:U}):s.createElement(f.B,{width:P,hide:!_,dataKey:m,axisLine:!1,tickLine:!1,ticks:T?[n[0][m],n[n.length-1][m]]:void 0,type:"category",interval:"preserveStartEnd",tick:{transform:"translate(0, 6)"},fill:"",stroke:"",className:(0,i.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content")}),s.createElement(b.u,{wrapperStyle:{outline:"none"},isAnimationActive:!1,cursor:{fill:"#d1d5db",opacity:"0.15"},content:M?e=>{let{active:t,payload:n,label:r}=e;return $?s.createElement($,{payload:null==n?void 0:n.map(e=>{var t;return Object.assign(Object.assign({},e),{color:null!==(t=J.get(e.dataKey))&&void 0!==t?t:o.fr.Gray})}),active:t,label:r}):s.createElement(E.ZP,{active:t,payload:n,label:r,valueFormatter:O,categoryColors:J})}:s.createElement(s.Fragment,null),position:{y:0}}),D?s.createElement(v.D,{verticalAlign:"top",height:X,content:e=>{let{payload:t}=e;return(0,y.Z)({payload:t},J,Q,en,eo?e=>{eo&&(e!==en||ee?(er(e),null==H||H({eventType:"category",categoryClicked:e})):(er(void 0),null==H||H(null)),et(void 0))}:void 0,G)}}):null,u.map(e=>{var t;return s.createElement(d.$,{className:(0,i.q)((0,l.bM)(null!==(t=J.get(e))&&void 0!==t?t:o.fr.Gray,a.K.background).fillColor,H?"cursor-pointer":""),key:e,name:e,type:"linear",stackId:C||A?"a":void 0,dataKey:e,fill:"",isAnimationActive:N,animationDuration:I,shape:e=>((e,t,n,r)=>{let{fillOpacity:o,name:a,payload:i,value:l}=e,{x:c,width:u,y:d,height:p}=e;return"horizontal"===r&&p<0?(d+=p,p=Math.abs(p)):"vertical"===r&&u<0&&(c+=u,u=Math.abs(u)),s.createElement("rect",{x:c,y:d,width:u,height:p,opacity:t||n&&n!==a?(0,w.vZ)(t,Object.assign(Object.assign({},i),{value:l}))?o:.3:o})})(e,ee,en,k),onClick:ea})})):s.createElement(S.Z,{noDataText:z})))});x.displayName="BarChart"},17280:function(e,t,n){n.d(t,{Z:function(){return g}});var r=n(64090);let o=(e,t)=>{let[n,o]=(0,r.useState)(t);(0,r.useEffect)(()=>{let t=()=>{o(window.innerWidth),e()};return t(),window.addEventListener("resize",t),()=>window.removeEventListener("resize",t)},[e,n])};var a=n(69703),i=n(2898),l=n(99250),s=n(65492);let c=e=>{var t=(0,a._T)(e,[]);return r.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"}),r.createElement("path",{d:"M8 12L14 6V18L8 12Z"}))},u=e=>{var t=(0,a._T)(e,[]);return r.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"}),r.createElement("path",{d:"M16 12L10 18V6L16 12Z"}))},d=(0,s.fn)("Legend"),p=e=>{let{name:t,color:n,onClick:o,activeLegend:a}=e,c=!!o;return r.createElement("li",{className:(0,l.q)(d("legendItem"),"group inline-flex items-center px-2 py-0.5 rounded-tremor-small transition whitespace-nowrap",c?"cursor-pointer":"cursor-default","text-tremor-content",c?"hover:bg-tremor-background-subtle":"","dark:text-dark-tremor-content",c?"dark:hover:bg-dark-tremor-background-subtle":""),onClick:e=>{e.stopPropagation(),null==o||o(t,n)}},r.createElement("svg",{className:(0,l.q)("flex-none h-2 w-2 mr-1.5",(0,s.bM)(n,i.K.text).textColor,a&&a!==t?"opacity-40":"opacity-100"),fill:"currentColor",viewBox:"0 0 8 8"},r.createElement("circle",{cx:4,cy:4,r:4})),r.createElement("p",{className:(0,l.q)("whitespace-nowrap truncate text-tremor-default","text-tremor-content",c?"group-hover:text-tremor-content-emphasis":"","dark:text-dark-tremor-content",a&&a!==t?"opacity-40":"opacity-100",c?"dark:group-hover:text-dark-tremor-content-emphasis":"")},t))},f=e=>{let{icon:t,onClick:n,disabled:o}=e,[a,i]=r.useState(!1),s=r.useRef(null);return r.useEffect(()=>(a?s.current=setInterval(()=>{null==n||n()},300):clearInterval(s.current),()=>clearInterval(s.current)),[a,n]),(0,r.useEffect)(()=>{o&&(clearInterval(s.current),i(!1))},[o]),r.createElement("button",{type:"button",className:(0,l.q)(d("legendSliderButton"),"w-5 group inline-flex items-center truncate rounded-tremor-small transition",o?"cursor-not-allowed":"cursor-pointer",o?"text-tremor-content-subtle":"text-tremor-content hover:text-tremor-content-emphasis hover:bg-tremor-background-subtle",o?"dark:text-dark-tremor-subtle":"dark:text-dark-tremor dark:hover:text-tremor-content-emphasis dark:hover:bg-dark-tremor-background-subtle"),disabled:o,onClick:e=>{e.stopPropagation(),null==n||n()},onMouseDown:e=>{e.stopPropagation(),i(!0)},onMouseUp:e=>{e.stopPropagation(),i(!1)}},r.createElement(t,{className:"w-full"}))},m=r.forwardRef((e,t)=>{var n,o;let{categories:s,colors:m=i.s,className:g,onClickLegendItem:h,activeLegend:b,enableLegendSlider:v=!1}=e,y=(0,a._T)(e,["categories","colors","className","onClickLegendItem","activeLegend","enableLegendSlider"]),E=r.useRef(null),[S,w]=r.useState(null),[x,O]=r.useState(null),k=r.useRef(null),C=(0,r.useCallback)(()=>{let e=null==E?void 0:E.current;e&&w({left:e.scrollLeft>0,right:e.scrollWidth-e.clientWidth>e.scrollLeft})},[w]),A=(0,r.useCallback)(e=>{var t;let n=null==E?void 0:E.current,r=null!==(t=null==n?void 0:n.clientWidth)&&void 0!==t?t:0;n&&v&&(n.scrollTo({left:"left"===e?n.scrollLeft-r:n.scrollLeft+r,behavior:"smooth"}),setTimeout(()=>{C()},400))},[v,C]);r.useEffect(()=>{let e=e=>{"ArrowLeft"===e?A("left"):"ArrowRight"===e&&A("right")};return x?(e(x),k.current=setInterval(()=>{e(x)},300)):clearInterval(k.current),()=>clearInterval(k.current)},[x,A]);let T=e=>{e.stopPropagation(),"ArrowLeft"!==e.key&&"ArrowRight"!==e.key||(e.preventDefault(),O(e.key))},I=e=>{e.stopPropagation(),O(null)};return r.useEffect(()=>{let e=null==E?void 0:E.current;return v&&(C(),null==e||e.addEventListener("keydown",T),null==e||e.addEventListener("keyup",I)),()=>{null==e||e.removeEventListener("keydown",T),null==e||e.removeEventListener("keyup",I)}},[C,v]),r.createElement("ol",Object.assign({ref:t,className:(0,l.q)(d("root"),"relative overflow-hidden",g)},y),r.createElement("div",{ref:E,tabIndex:0,className:(0,l.q)("h-full flex",v?(null==S?void 0:S.right)||(null==S?void 0:S.left)?"pl-4 pr-12 items-center overflow-auto snap-mandatory [&::-webkit-scrollbar]:hidden [scrollbar-width:none]":"":"flex-wrap")},s.map((e,t)=>r.createElement(p,{key:"item-".concat(t),name:e,color:m[t],onClick:h,activeLegend:b}))),v&&((null==S?void 0:S.right)||(null==S?void 0:S.left))?r.createElement(r.Fragment,null,r.createElement("div",{className:(0,l.q)("from-tremor-background","dark:from-dark-tremor-background","absolute top-0 bottom-0 left-0 w-4 bg-gradient-to-r to-transparent pointer-events-none")}),r.createElement("div",{className:(0,l.q)("to-tremor-background","dark:to-dark-tremor-background","absolute top-0 bottom-0 right-10 w-4 bg-gradient-to-r from-transparent pointer-events-none")}),r.createElement("div",{className:(0,l.q)("bg-tremor-background","dark:bg-dark-tremor-background","absolute flex top-0 pr-1 bottom-0 right-0 items-center justify-center h-full")},r.createElement(f,{icon:c,onClick:()=>{O(null),A("left")},disabled:!(null==S?void 0:S.left)}),r.createElement(f,{icon:u,onClick:()=>{O(null),A("right")},disabled:!(null==S?void 0:S.right)}))):null)});m.displayName="Legend";let g=(e,t,n,a,i,l)=>{let{payload:s}=e,c=(0,r.useRef)(null);o(()=>{var e,t;n((t=null===(e=c.current)||void 0===e?void 0:e.clientHeight)?Number(t)+20:60)});let u=s.filter(e=>"none"!==e.type);return r.createElement("div",{ref:c,className:"flex items-center justify-end"},r.createElement(m,{categories:u.map(e=>e.value),colors:u.map(e=>t.get(e.value)),onClickLegendItem:i,activeLegend:a,enableLegendSlider:l}))}},30470:function(e,t,n){n.d(t,{ZP:function(){return u}});var r=n(64090),o=n(54942),a=n(2898),i=n(99250),l=n(65492);let s=e=>{let{children:t}=e;return r.createElement("div",{className:(0,i.q)("rounded-tremor-default text-tremor-default border","bg-tremor-background shadow-tremor-dropdown border-tremor-border","dark:bg-dark-tremor-background dark:shadow-dark-tremor-dropdown dark:border-dark-tremor-border")},t)},c=e=>{let{value:t,name:n,color:o}=e;return r.createElement("div",{className:"flex items-center justify-between space-x-8"},r.createElement("div",{className:"flex items-center space-x-2"},r.createElement("span",{className:(0,i.q)("shrink-0 rounded-tremor-full border-2 h-3 w-3","border-tremor-background shadow-tremor-card","dark:border-dark-tremor-background dark:shadow-dark-tremor-card",(0,l.bM)(o,a.K.background).bgColor)}),r.createElement("p",{className:(0,i.q)("text-right whitespace-nowrap","text-tremor-content","dark:text-dark-tremor-content")},n)),r.createElement("p",{className:(0,i.q)("font-medium tabular-nums text-right whitespace-nowrap","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis")},t))},u=e=>{let{active:t,payload:n,label:a,categoryColors:l,valueFormatter:u}=e;if(t&&n){let e=n.filter(e=>"none"!==e.type);return r.createElement(s,null,r.createElement("div",{className:(0,i.q)("border-tremor-border border-b px-4 py-2","dark:border-dark-tremor-border")},r.createElement("p",{className:(0,i.q)("font-medium","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis")},a)),r.createElement("div",{className:(0,i.q)("px-4 py-2 space-y-1")},e.map((e,t)=>{var n;let{value:a,name:i}=e;return r.createElement(c,{key:"id-".concat(t),value:u(a),name:i,color:null!==(n=l.get(i))&&void 0!==n?n:o.fr.Blue})})))}return null}},77448:function(e,t,n){n.d(t,{Z:function(){return p}});var r=n(99250),o=n(64090),a=n(69703);let i=(0,n(65492).fn)("Flex"),l={start:"justify-start",end:"justify-end",center:"justify-center",between:"justify-between",around:"justify-around",evenly:"justify-evenly"},s={start:"items-start",end:"items-end",center:"items-center",baseline:"items-baseline",stretch:"items-stretch"},c={row:"flex-row",col:"flex-col","row-reverse":"flex-row-reverse","col-reverse":"flex-col-reverse"},u=o.forwardRef((e,t)=>{let{flexDirection:n="row",justifyContent:u="between",alignItems:d="center",children:p,className:f}=e,m=(0,a._T)(e,["flexDirection","justifyContent","alignItems","children","className"]);return o.createElement("div",Object.assign({ref:t,className:(0,r.q)(i("root"),"flex w-full",c[n],l[u],s[d],f)},m),p)});u.displayName="Flex";var d=n(71801);let p=e=>{let{noDataText:t="No data"}=e;return o.createElement(u,{alignItems:"center",justifyContent:"center",className:(0,r.q)("w-full h-full border border-dashed rounded-tremor-default","border-tremor-border","dark:border-dark-tremor-border")},o.createElement(d.Z,{className:(0,r.q)("text-tremor-content","dark:text-dark-tremor-content")},t))}},36342:function(e,t,n){n.d(t,{FB:function(){return a},i4:function(){return o},me:function(){return r},vZ:function(){return function e(t,n){if(t===n)return!0;if("object"!=typeof t||"object"!=typeof n||null===t||null===n)return!1;let r=Object.keys(t),o=Object.keys(n);if(r.length!==o.length)return!1;for(let a of r)if(!o.includes(a)||!e(t[a],n[a]))return!1;return!0}}});let r=(e,t)=>{let n=new Map;return e.forEach((e,r)=>{n.set(e,t[r])}),n},o=(e,t,n)=>[e?"auto":null!=t?t:0,null!=n?n:"auto"];function a(e,t){let n=[];for(let r of e)if(Object.prototype.hasOwnProperty.call(r,t)&&(n.push(r[t]),n.length>1))return!1;return!0}},5:function(e,t,n){n.d(t,{Z:function(){return f}});var r=n(69703),o=n(64090),a=n(58437),i=n(54942),l=n(2898),s=n(99250),c=n(65492);let u={xs:{paddingX:"px-2",paddingY:"py-0.5",fontSize:"text-xs"},sm:{paddingX:"px-2.5",paddingY:"py-0.5",fontSize:"text-sm"},md:{paddingX:"px-3",paddingY:"py-0.5",fontSize:"text-md"},lg:{paddingX:"px-3.5",paddingY:"py-0.5",fontSize:"text-lg"},xl:{paddingX:"px-4",paddingY:"py-1",fontSize:"text-xl"}},d={xs:{height:"h-4",width:"w-4"},sm:{height:"h-4",width:"w-4"},md:{height:"h-4",width:"w-4"},lg:{height:"h-5",width:"w-5"},xl:{height:"h-6",width:"w-6"}},p=(0,c.fn)("Badge"),f=o.forwardRef((e,t)=>{let{color:n,icon:f,size:m=i.u8.SM,tooltip:g,className:h,children:b}=e,v=(0,r._T)(e,["color","icon","size","tooltip","className","children"]),y=f||null,{tooltipProps:E,getReferenceProps:S}=(0,a.l)();return o.createElement("span",Object.assign({ref:(0,c.lq)([t,E.refs.setReference]),className:(0,s.q)(p("root"),"w-max flex-shrink-0 inline-flex justify-center items-center cursor-default rounded-tremor-full",n?(0,s.q)((0,c.bM)(n,l.K.background).bgColor,(0,c.bM)(n,l.K.text).textColor,"bg-opacity-20 dark:bg-opacity-25"):(0,s.q)("bg-tremor-brand-muted text-tremor-brand-emphasis","dark:bg-dark-tremor-brand-muted dark:text-dark-tremor-brand-emphasis"),u[m].paddingX,u[m].paddingY,u[m].fontSize,h)},S,v),o.createElement(a.Z,Object.assign({text:g},E)),y?o.createElement(y,{className:(0,s.q)(p("icon"),"shrink-0 -ml-1 mr-1.5",d[m].height,d[m].width)}):null,o.createElement("p",{className:(0,s.q)(p("text"),"text-sm whitespace-nowrap")},b))});f.displayName="Badge"},61244:function(e,t,n){n.d(t,{Z:function(){return g}});var r=n(69703),o=n(64090),a=n(58437),i=n(54942),l=n(99250),s=n(65492),c=n(2898);let u={xs:{paddingX:"px-1.5",paddingY:"py-1.5"},sm:{paddingX:"px-1.5",paddingY:"py-1.5"},md:{paddingX:"px-2",paddingY:"py-2"},lg:{paddingX:"px-2",paddingY:"py-2"},xl:{paddingX:"px-2.5",paddingY:"py-2.5"}},d={xs:{height:"h-3",width:"w-3"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-7",width:"w-7"},xl:{height:"h-9",width:"w-9"}},p={simple:{rounded:"",border:"",ring:"",shadow:""},light:{rounded:"rounded-tremor-default",border:"",ring:"",shadow:""},shadow:{rounded:"rounded-tremor-default",border:"border",ring:"",shadow:"shadow-tremor-card dark:shadow-dark-tremor-card"},solid:{rounded:"rounded-tremor-default",border:"border-2",ring:"ring-1",shadow:""},outlined:{rounded:"rounded-tremor-default",border:"border",ring:"ring-2",shadow:""}},f=(e,t)=>{switch(e){case"simple":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:"",borderColor:"",ringColor:""};case"light":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-brand-muted dark:bg-dark-tremor-brand-muted",borderColor:"",ringColor:""};case"shadow":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:"border-tremor-border dark:border-dark-tremor-border",ringColor:""};case"solid":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-brand dark:bg-dark-tremor-brand",borderColor:"border-tremor-brand-inverted dark:border-dark-tremor-brand-inverted",ringColor:"ring-tremor-ring dark:ring-dark-tremor-ring"};case"outlined":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:t?(0,s.bM)(t,c.K.ring).borderColor:"border-tremor-brand-subtle dark:border-dark-tremor-brand-subtle",ringColor:t?(0,l.q)((0,s.bM)(t,c.K.ring).ringColor,"ring-opacity-40"):"ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted"}}},m=(0,s.fn)("Icon"),g=o.forwardRef((e,t)=>{let{icon:n,variant:c="simple",tooltip:g,size:h=i.u8.SM,color:b,className:v}=e,y=(0,r._T)(e,["icon","variant","tooltip","size","color","className"]),E=f(c,b),{tooltipProps:S,getReferenceProps:w}=(0,a.l)();return o.createElement("span",Object.assign({ref:(0,s.lq)([t,S.refs.setReference]),className:(0,l.q)(m("root"),"inline-flex flex-shrink-0 items-center",E.bgColor,E.textColor,E.borderColor,E.ringColor,p[c].rounded,p[c].border,p[c].shadow,p[c].ring,u[h].paddingX,u[h].paddingY,v)},w,y),o.createElement(a.Z,Object.assign({text:g},S)),o.createElement(n,{className:(0,l.q)(m("icon"),"shrink-0",d[h].height,d[h].width)}))});g.displayName="Icon"},2179:function(e,t,n){n.d(t,{Z:function(){return O}});var r=n(69703),o=n(58437),a=n(64090);let i=["preEnter","entering","entered","preExit","exiting","exited","unmounted"],l=e=>({_s:e,status:i[e],isEnter:e<3,isMounted:6!==e,isResolved:2===e||e>4}),s=e=>e?6:5,c=(e,t)=>{switch(e){case 1:case 0:return 2;case 4:case 3:return s(t)}},u=e=>"object"==typeof e?[e.enter,e.exit]:[e,e],d=(e,t)=>setTimeout(()=>{isNaN(document.body.offsetTop)||e(t+1)},0),p=(e,t,n,r,o)=>{clearTimeout(r.current);let a=l(e);t(a),n.current=a,o&&o({current:a})},f=function(){let{enter:e=!0,exit:t=!0,preEnter:n,preExit:r,timeout:o,initialEntered:i,mountOnEnter:f,unmountOnExit:m,onStateChange:g}=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},[h,b]=(0,a.useState)(()=>l(i?2:s(f))),v=(0,a.useRef)(h),y=(0,a.useRef)(),[E,S]=u(o),w=(0,a.useCallback)(()=>{let e=c(v.current._s,m);e&&p(e,b,v,y,g)},[g,m]),x=(0,a.useCallback)(o=>{let a=e=>{switch(p(e,b,v,y,g),e){case 1:E>=0&&(y.current=setTimeout(w,E));break;case 4:S>=0&&(y.current=setTimeout(w,S));break;case 0:case 3:y.current=d(a,e)}},i=v.current.isEnter;"boolean"!=typeof o&&(o=!i),o?i||a(e?n?0:1:2):i&&a(t?r?3:4:s(m))},[w,g,e,t,n,r,E,S,m]);return(0,a.useEffect)(()=>()=>clearTimeout(y.current),[]),[h,x,w]};var m=n(54942),g=n(99250),h=n(65492);let b=e=>{var t=(0,r._T)(e,[]);return a.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"}),a.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),a.createElement("path",{d:"M18.364 5.636L16.95 7.05A7 7 0 1 0 19 12h2a9 9 0 1 1-2.636-6.364z"}))};var v=n(2898);let y={xs:{height:"h-4",width:"w-4"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-6",width:"w-6"},xl:{height:"h-6",width:"w-6"}},E=e=>"light"!==e?{xs:{paddingX:"px-2.5",paddingY:"py-1.5",fontSize:"text-xs"},sm:{paddingX:"px-4",paddingY:"py-2",fontSize:"text-sm"},md:{paddingX:"px-4",paddingY:"py-2",fontSize:"text-md"},lg:{paddingX:"px-4",paddingY:"py-2.5",fontSize:"text-lg"},xl:{paddingX:"px-4",paddingY:"py-3",fontSize:"text-xl"}}:{xs:{paddingX:"",paddingY:"",fontSize:"text-xs"},sm:{paddingX:"",paddingY:"",fontSize:"text-sm"},md:{paddingX:"",paddingY:"",fontSize:"text-md"},lg:{paddingX:"",paddingY:"",fontSize:"text-lg"},xl:{paddingX:"",paddingY:"",fontSize:"text-xl"}},S=(e,t)=>{switch(e){case"primary":return{textColor:t?(0,h.bM)("white").textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",hoverTextColor:t?(0,h.bM)("white").textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,h.bM)(t,v.K.background).bgColor:"bg-tremor-brand dark:bg-dark-tremor-brand",hoverBgColor:t?(0,h.bM)(t,v.K.darkBackground).hoverBgColor:"hover:bg-tremor-brand-emphasis dark:hover:bg-dark-tremor-brand-emphasis",borderColor:t?(0,h.bM)(t,v.K.border).borderColor:"border-tremor-brand dark:border-dark-tremor-brand",hoverBorderColor:t?(0,h.bM)(t,v.K.darkBorder).hoverBorderColor:"hover:border-tremor-brand-emphasis dark:hover:border-dark-tremor-brand-emphasis"};case"secondary":return{textColor:t?(0,h.bM)(t,v.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",hoverTextColor:t?(0,h.bM)(t,v.K.text).textColor:"hover:text-tremor-brand-emphasis dark:hover:text-dark-tremor-brand-emphasis",bgColor:(0,h.bM)("transparent").bgColor,hoverBgColor:t?(0,g.q)((0,h.bM)(t,v.K.background).hoverBgColor,"hover:bg-opacity-20 dark:hover:bg-opacity-20"):"hover:bg-tremor-brand-faint dark:hover:bg-dark-tremor-brand-faint",borderColor:t?(0,h.bM)(t,v.K.border).borderColor:"border-tremor-brand dark:border-dark-tremor-brand"};case"light":return{textColor:t?(0,h.bM)(t,v.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",hoverTextColor:t?(0,h.bM)(t,v.K.darkText).hoverTextColor:"hover:text-tremor-brand-emphasis dark:hover:text-dark-tremor-brand-emphasis",bgColor:(0,h.bM)("transparent").bgColor,borderColor:"",hoverBorderColor:""}}},w=(0,h.fn)("Button"),x=e=>{let{loading:t,iconSize:n,iconPosition:r,Icon:o,needMargin:i,transitionStatus:l}=e,s=i?r===m.zS.Left?(0,g.q)("-ml-1","mr-1.5"):(0,g.q)("-mr-1","ml-1.5"):"",c=(0,g.q)("w-0 h-0"),u={default:c,entering:c,entered:n,exiting:n,exited:c};return t?a.createElement(b,{className:(0,g.q)(w("icon"),"animate-spin shrink-0",s,u.default,u[l]),style:{transition:"width 150ms"}}):a.createElement(o,{className:(0,g.q)(w("icon"),"shrink-0",n,s)})},O=a.forwardRef((e,t)=>{let{icon:n,iconPosition:i=m.zS.Left,size:l=m.u8.SM,color:s,variant:c="primary",disabled:u,loading:d=!1,loadingText:p,children:b,tooltip:v,className:O}=e,k=(0,r._T)(e,["icon","iconPosition","size","color","variant","disabled","loading","loadingText","children","tooltip","className"]),C=d||u,A=void 0!==n||d,T=d&&p,I=!(!b&&!T),N=(0,g.q)(y[l].height,y[l].width),R="light"!==c?(0,g.q)("rounded-tremor-default border","shadow-tremor-input","dark:shadow-dark-tremor-input"):"",_=S(c,s),P=E(c)[l],{tooltipProps:L,getReferenceProps:M}=(0,o.l)(300),[D,j]=f({timeout:50});return(0,a.useEffect)(()=>{j(d)},[d]),a.createElement("button",Object.assign({ref:(0,h.lq)([t,L.refs.setReference]),className:(0,g.q)(w("root"),"flex-shrink-0 inline-flex justify-center items-center group font-medium outline-none",R,P.paddingX,P.paddingY,P.fontSize,_.textColor,_.bgColor,_.borderColor,_.hoverBorderColor,C?"opacity-50 cursor-not-allowed":(0,g.q)(S(c,s).hoverTextColor,S(c,s).hoverBgColor,S(c,s).hoverBorderColor),O),disabled:C},M,k),a.createElement(o.Z,Object.assign({text:v},L)),A&&i!==m.zS.Right?a.createElement(x,{loading:d,iconSize:N,iconPosition:i,Icon:n,transitionStatus:D.status,needMargin:I}):null,T||b?a.createElement("span",{className:(0,g.q)(w("text"),"text-tremor-default whitespace-nowrap")},T?p:b):null,A&&i===m.zS.Right?a.createElement(x,{loading:d,iconSize:N,iconPosition:i,Icon:n,transitionStatus:D.status,needMargin:I}):null)});O.displayName="Button"},47047:function(e,t,n){n.d(t,{Z:function(){return b}});var r=n(69703),o=n(64090);n(50027),n(18174),n(21871);var a=n(41213),i=n(46457),l=n(54518);let s=e=>{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"},t),o.createElement("path",{d:"M18.031 16.6168L22.3137 20.8995L20.8995 22.3137L16.6168 18.031C15.0769 19.263 13.124 20 11 20C6.032 20 2 15.968 2 11C2 6.032 6.032 2 11 2C15.968 2 20 6.032 20 11C20 13.124 19.263 15.0769 18.031 16.6168ZM16.0247 15.8748C17.2475 14.6146 18 12.8956 18 11C18 7.1325 14.8675 4 11 4C7.1325 4 4 7.1325 4 11C4 14.8675 7.1325 18 11 18C12.8956 18 14.6146 17.2475 15.8748 16.0247L16.0247 15.8748Z"}))};var c=n(8903),u=n(25163),d=n(70129);let p=e=>{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",width:"100%",height:"100%",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2",strokeLinecap:"round",strokeLinejoin:"round"},t),o.createElement("line",{x1:"18",y1:"6",x2:"6",y2:"18"}),o.createElement("line",{x1:"6",y1:"6",x2:"18",y2:"18"}))};var f=n(99250),m=n(65492),g=n(91753);let h=(0,m.fn)("MultiSelect"),b=o.forwardRef((e,t)=>{let{defaultValue:n,value:m,onValueChange:b,placeholder:v="Select...",placeholderSearch:y="Search",disabled:E=!1,icon:S,children:w,className:x}=e,O=(0,r._T)(e,["defaultValue","value","onValueChange","placeholder","placeholderSearch","disabled","icon","children","className"]),[k,C]=(0,i.Z)(n,m),{reactElementChildren:A,optionsAvailable:T}=(0,o.useMemo)(()=>{let e=o.Children.toArray(w).filter(o.isValidElement);return{reactElementChildren:e,optionsAvailable:(0,g.n0)("",e)}},[w]),[I,N]=(0,o.useState)(""),R=(null!=k?k:[]).length>0,_=(0,o.useMemo)(()=>I?(0,g.n0)(I,A):T,[I,A,T]),P=()=>{N("")};return o.createElement(u.R,Object.assign({as:"div",ref:t,defaultValue:k,value:k,onChange:e=>{null==b||b(e),C(e)},disabled:E,className:(0,f.q)("w-full min-w-[10rem] relative text-tremor-default",x)},O,{multiple:!0}),e=>{let{value:t}=e;return o.createElement(o.Fragment,null,o.createElement(u.R.Button,{className:(0,f.q)("w-full outline-none text-left whitespace-nowrap truncate rounded-tremor-default focus:ring-2 transition duration-100 border pr-8 py-1.5","border-tremor-border shadow-tremor-input focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",S?"pl-11 -ml-0.5":"pl-3",(0,g.um)(t.length>0,E))},S&&o.createElement("span",{className:(0,f.q)("absolute inset-y-0 left-0 flex items-center ml-px pl-2.5")},o.createElement(S,{className:(0,f.q)(h("Icon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),o.createElement("div",{className:"h-6 flex items-center"},t.length>0?o.createElement("div",{className:"flex flex-nowrap overflow-x-scroll [&::-webkit-scrollbar]:hidden [scrollbar-width:none] gap-x-1 mr-5 -ml-1.5 relative"},T.filter(e=>t.includes(e.props.value)).map((e,n)=>{var r;return o.createElement("div",{key:n,className:(0,f.q)("max-w-[100px] lg:max-w-[200px] flex justify-center items-center pl-2 pr-1.5 py-1 font-medium","rounded-tremor-small","bg-tremor-background-muted dark:bg-dark-tremor-background-muted","bg-tremor-background-subtle dark:bg-dark-tremor-background-subtle","text-tremor-content-default dark:text-dark-tremor-content-default","text-tremor-content-emphasis dark:text-dark-tremor-content-emphasis")},o.createElement("div",{className:"text-xs truncate "},null!==(r=e.props.children)&&void 0!==r?r:e.props.value),o.createElement("div",{onClick:n=>{n.preventDefault();let r=t.filter(t=>t!==e.props.value);null==b||b(r),C(r)}},o.createElement(p,{className:(0,f.q)(h("clearIconItem"),"cursor-pointer rounded-tremor-full w-3.5 h-3.5 ml-2","text-tremor-content-subtle hover:text-tremor-content","dark:text-dark-tremor-content-subtle dark:hover:text-tremor-content")})))})):o.createElement("span",null,v)),o.createElement("span",{className:(0,f.q)("absolute inset-y-0 right-0 flex items-center mr-2.5")},o.createElement(l.Z,{className:(0,f.q)(h("arrowDownIcon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}))),R&&!E?o.createElement("button",{type:"button",className:(0,f.q)("absolute inset-y-0 right-0 flex items-center mr-8"),onClick:e=>{e.preventDefault(),C([]),null==b||b([])}},o.createElement(c.Z,{className:(0,f.q)(h("clearIconAllItems"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null,o.createElement(d.u,{className:"absolute z-10 w-full",enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},o.createElement(u.R.Options,{className:(0,f.q)("divide-y overflow-y-auto outline-none rounded-tremor-default max-h-[228px] left-0 border my-1","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},o.createElement("div",{className:(0,f.q)("flex items-center w-full px-2.5","bg-tremor-background-muted","dark:bg-dark-tremor-background-muted")},o.createElement("span",null,o.createElement(s,{className:(0,f.q)("flex-none w-4 h-4 mr-2","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),o.createElement("input",{name:"search",type:"input",autoComplete:"off",placeholder:y,className:(0,f.q)("w-full focus:outline-none focus:ring-none bg-transparent text-tremor-default py-2","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis"),onKeyDown:e=>{"Space"===e.code&&""!==e.target.value&&e.stopPropagation()},onChange:e=>N(e.target.value),value:I})),o.createElement(a.Z.Provider,Object.assign({},{onBlur:{handleResetSearch:P}},{value:{selectedValue:t}}),_))))})});b.displayName="MultiSelect"},76628:function(e,t,n){n.d(t,{Z:function(){return u}});var r=n(69703);n(50027),n(18174),n(21871);var o=n(41213),a=n(64090),i=n(99250),l=n(65492),s=n(25163);let c=(0,l.fn)("MultiSelectItem"),u=a.forwardRef((e,t)=>{let{value:n,className:u,children:d}=e,p=(0,r._T)(e,["value","className","children"]),{selectedValue:f}=(0,a.useContext)(o.Z),m=(0,l.NZ)(n,f);return a.createElement(s.R.Option,Object.assign({className:(0,i.q)(c("root"),"flex justify-start items-center cursor-default text-tremor-default p-2.5","ui-active:bg-tremor-background-muted ui-active:text-tremor-content-strong ui-selected:text-tremor-content-strong text-tremor-content-emphasis","dark:ui-active:bg-dark-tremor-background-muted dark:ui-active:text-dark-tremor-content-strong dark:ui-selected:text-dark-tremor-content-strong dark:ui-selected:bg-dark-tremor-background-muted dark:text-dark-tremor-content-emphasis",u),ref:t,key:n,value:n},p),a.createElement("input",{type:"checkbox",className:(0,i.q)(c("checkbox"),"flex-none focus:ring-none focus:outline-none cursor-pointer mr-2.5","accent-tremor-brand","dark:accent-dark-tremor-brand"),checked:m,readOnly:!0}),a.createElement("span",{className:"whitespace-nowrap truncate"},null!=d?d:n))});u.displayName="MultiSelectItem"},95093:function(e,t,n){n.d(t,{Z:function(){return m}});var r=n(69703),o=n(64090),a=n(54518),i=n(8903),l=n(99250),s=n(65492),c=n(91753),u=n(25163),d=n(70129),p=n(46457);let f=(0,s.fn)("Select"),m=o.forwardRef((e,t)=>{let{defaultValue:n,value:s,onValueChange:m,placeholder:g="Select...",disabled:h=!1,icon:b,enableClear:v=!0,children:y,className:E}=e,S=(0,r._T)(e,["defaultValue","value","onValueChange","placeholder","disabled","icon","enableClear","children","className"]),[w,x]=(0,p.Z)(n,s),O=(0,o.useMemo)(()=>{let e=o.Children.toArray(y).filter(o.isValidElement);return(0,c.sl)(e)},[y]);return o.createElement(u.R,Object.assign({as:"div",ref:t,defaultValue:w,value:w,onChange:e=>{null==m||m(e),x(e)},disabled:h,className:(0,l.q)("w-full min-w-[10rem] relative text-tremor-default",E)},S),e=>{var t;let{value:n}=e;return o.createElement(o.Fragment,null,o.createElement(u.R.Button,{className:(0,l.q)("w-full outline-none text-left whitespace-nowrap truncate rounded-tremor-default focus:ring-2 transition duration-100 border pr-8 py-2","border-tremor-border shadow-tremor-input focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",b?"pl-10":"pl-3",(0,c.um)((0,c.Uh)(n),h))},b&&o.createElement("span",{className:(0,l.q)("absolute inset-y-0 left-0 flex items-center ml-px pl-2.5")},o.createElement(b,{className:(0,l.q)(f("Icon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),o.createElement("span",{className:"w-[90%] block truncate"},n&&null!==(t=O.get(n))&&void 0!==t?t:g),o.createElement("span",{className:(0,l.q)("absolute inset-y-0 right-0 flex items-center mr-3")},o.createElement(a.Z,{className:(0,l.q)(f("arrowDownIcon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}))),v&&w?o.createElement("button",{type:"button",className:(0,l.q)("absolute inset-y-0 right-0 flex items-center mr-8"),onClick:e=>{e.preventDefault(),x(""),null==m||m("")}},o.createElement(i.Z,{className:(0,l.q)(f("clearIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null,o.createElement(d.u,{className:"absolute z-10 w-full",enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},o.createElement(u.R.Options,{className:(0,l.q)("divide-y overflow-y-auto outline-none rounded-tremor-default max-h-[228px] left-0 border my-1","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},y)))})});m.displayName="Select"},27166:function(e,t,n){n.d(t,{Z:function(){return s}});var r=n(69703),o=n(64090),a=n(25163),i=n(99250);let l=(0,n(65492).fn)("SelectItem"),s=o.forwardRef((e,t)=>{let{value:n,icon:s,className:c,children:u}=e,d=(0,r._T)(e,["value","icon","className","children"]);return o.createElement(a.R.Option,Object.assign({className:(0,i.q)(l("root"),"flex justify-start items-center cursor-default text-tremor-default px-2.5 py-2.5","ui-active:bg-tremor-background-muted ui-active:text-tremor-content-strong ui-selected:text-tremor-content-strong ui-selected:bg-tremor-background-muted text-tremor-content-emphasis","dark:ui-active:bg-dark-tremor-background-muted dark:ui-active:text-dark-tremor-content-strong dark:ui-selected:text-dark-tremor-content-strong dark:ui-selected:bg-dark-tremor-background-muted dark:text-dark-tremor-content-emphasis",c),ref:t,key:n,value:n},d),s&&o.createElement(s,{className:(0,i.q)(l("icon"),"flex-none w-5 h-5 mr-1.5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}),o.createElement("span",{className:"whitespace-nowrap truncate"},null!=u?u:n))});s.displayName="SelectItem"},12224:function(e,t,n){n.d(t,{Z:function(){return I}});var r=n(69703),o=n(64090),a=n(83891),i=n(20044),l=n(10641),s=n(92381),c=n(71454),u=n(36601),d=n(37700),p=n(84152),f=n(34797),m=n(18318),g=n(71014),h=n(67409),b=n(39790);let v=(0,o.createContext)(null),y=Object.assign((0,m.yV)(function(e,t){let n=(0,s.M)(),{id:r="headlessui-label-".concat(n),passive:a=!1,...i}=e,l=function e(){let t=(0,o.useContext)(v);if(null===t){let t=Error("You used a