diff --git a/.circleci/config.yml b/.circleci/config.yml index 516f2b20d..5f4628d26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 4.3.4 jobs: local_testing: docker: - - image: circleci/python:3.9 + - image: cimg/python:3.11 working_directory: ~/project steps: @@ -43,7 +43,10 @@ jobs: pip install "langfuse==2.27.1" pip install "logfire==0.29.0" pip install numpydoc - pip install traceloop-sdk==0.18.2 + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 pip install openai pip install prisma pip install "httpx==0.24.1" @@ -61,6 +64,7 @@ jobs: pip install prometheus-client==0.20.0 pip install "pydantic==2.7.1" pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" - save_cache: paths: - ./venv @@ -340,4 +344,4 @@ workflows: filters: branches: only: - - main + - main \ No newline at end of file diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index b505536e2..c4225a9aa 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -7,6 +7,5 @@ cohere redis anthropic orjson -pydantic==1.10.14 +pydantic==2.7.1 google-cloud-aiplatform==1.43.0 -redisvl==0.0.7 # semantic caching \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..23e4a06da --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,34 @@ +name: Publish Dev Release to PyPI + +on: + workflow_dispatch: + +jobs: + publish-dev-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 # Adjust the Python version as needed + + - name: Install dependencies + run: pip install toml twine + + - name: Read version from pyproject.toml + id: read-version + run: | + version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') + printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV + + - name: Check if version exists on PyPI + id: check-version + run: | + set -e + if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then + echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish." + diff --git a/.gitignore b/.gitignore index b75a92309..69061d62d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,8 @@ litellm/proxy/_super_secret_config.yaml litellm/proxy/_super_secret_config.yaml litellm/proxy/myenv/bin/activate litellm/proxy/myenv/bin/Activate.ps1 -myenv/* \ No newline at end of file +myenv/* +litellm/proxy/_experimental/out/404/index.html +litellm/proxy/_experimental/out/model_hub/index.html +litellm/proxy/_experimental/out/onboarding/index.html +litellm/tests/log.txt diff --git a/README.md b/README.md index 415ea8480..fe7d56b6c 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ The proxy provides: ## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/) + ## Quick Start Proxy - CLI ```shell @@ -179,6 +180,24 @@ print(response) ## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys)) +Connect the proxy with a Postgres DB to create proxy keys + +```bash +# Get the code +git clone https://github.com/BerriAI/litellm + +# Go to folder +cd litellm + +# Add the master key +echo 'LITELLM_MASTER_KEY="sk-1234"' > .env +source .env + +# Start +docker-compose up +``` + + UI on `/ui` on your proxy server ![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) @@ -206,37 +225,37 @@ curl 'http://0.0.0.0:4000/key/generate' \ ## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers)) | Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) | -| ----------------------------------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | -| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | -| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | -| [google - vertex_ai [Gemini]](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | -| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | -| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | -| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | -| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | -| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | -| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | -| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | -| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | -| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | -| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | -| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | -| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | -| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | -| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | -| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | -| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | -| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | -| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | -| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | -| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | -| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | -| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | -| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ -| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | -| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | +|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------| +| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | | | +| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | | +| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | | +| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | | +| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | | +| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | | +| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | | | +| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | | | +| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | | | +| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | | | +| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | | | +| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | | | +| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | | | +| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | | +| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | | +| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | | +| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | | +| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | | +| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | | +| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | | [**Read the Docs**](https://docs.litellm.ai/docs/) diff --git a/docker-compose.yml b/docker-compose.yml index 05439b1df..6c1f5f57b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,29 @@ -version: "3.9" +version: "3.11" services: litellm: build: context: . args: target: runtime - image: ghcr.io/berriai/litellm:main-latest + image: ghcr.io/berriai/litellm:main-stable ports: - "4000:4000" # Map the container port to the host, change the host port if necessary - volumes: - - ./litellm-config.yaml:/app/config.yaml # Mount the local configuration file - # You can change the port or number of workers as per your requirements or pass any new supported CLI augument. Make sure the port passed here matches with the container port defined above in `ports` value - command: [ "--config", "/app/config.yaml", "--port", "4000", "--num_workers", "8" ] + environment: + DATABASE_URL: "postgresql://postgres:example@db:5432/postgres" + STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI + env_file: + - .env # Load local .env file + + + db: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: example + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 5s + retries: 10 # ...rest of your docker-compose config if any \ No newline at end of file diff --git a/docs/my-website/docs/assistants.md b/docs/my-website/docs/assistants.md new file mode 100644 index 000000000..1af780500 --- /dev/null +++ b/docs/my-website/docs/assistants.md @@ -0,0 +1,238 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Assistants API + +Covers Threads, Messages, Assistants. + +LiteLLM currently covers: +- Get Assistants +- Create Thread +- Get Thread +- Add Messages +- Get Messages +- Run Thread + +## Quick Start + +Call an existing Assistant. + +- Get the Assistant + +- Create a Thread when a user starts a conversation. + +- Add Messages to the Thread as the user asks questions. + +- Run the Assistant on the Thread to generate a response by calling the model and the tools. + + + + +**Get the Assistant** + +```python +from litellm import get_assistants, aget_assistants +import os + +# setup env +os.environ["OPENAI_API_KEY"] = "sk-.." + +assistants = get_assistants(custom_llm_provider="openai") + +### ASYNC USAGE ### +# assistants = await aget_assistants(custom_llm_provider="openai") +``` + +**Create a Thread** + +```python +from litellm import create_thread, acreate_thread +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +new_thread = create_thread( + custom_llm_provider="openai", + messages=[{"role": "user", "content": "Hey, how's it going?"}], # type: ignore + ) + +### ASYNC USAGE ### +# new_thread = await acreate_thread(custom_llm_provider="openai",messages=[{"role": "user", "content": "Hey, how's it going?"}]) +``` + +**Add Messages to the Thread** + +```python +from litellm import create_thread, get_thread, aget_thread, add_message, a_add_message +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +## CREATE A THREAD +_new_thread = create_thread( + custom_llm_provider="openai", + messages=[{"role": "user", "content": "Hey, how's it going?"}], # type: ignore + ) + +## OR retrieve existing thread +received_thread = get_thread( + custom_llm_provider="openai", + thread_id=_new_thread.id, + ) + +### ASYNC USAGE ### +# received_thread = await aget_thread(custom_llm_provider="openai", thread_id=_new_thread.id,) + +## ADD MESSAGE TO THREAD +message = {"role": "user", "content": "Hey, how's it going?"} +added_message = add_message( + thread_id=_new_thread.id, custom_llm_provider="openai", **message + ) + +### ASYNC USAGE ### +# added_message = await a_add_message(thread_id=_new_thread.id, custom_llm_provider="openai", **message) +``` + +**Run the Assistant on the Thread** + +```python +from litellm import get_assistants, create_thread, add_message, run_thread, arun_thread +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." +assistants = get_assistants(custom_llm_provider="openai") + +## get the first assistant ### +assistant_id = assistants.data[0].id + +## GET A THREAD +_new_thread = create_thread( + custom_llm_provider="openai", + messages=[{"role": "user", "content": "Hey, how's it going?"}], # type: ignore + ) + +## ADD MESSAGE +message = {"role": "user", "content": "Hey, how's it going?"} +added_message = add_message( + thread_id=_new_thread.id, custom_llm_provider="openai", **message + ) + +## 🚨 RUN THREAD +response = run_thread( + custom_llm_provider="openai", thread_id=thread_id, assistant_id=assistant_id + ) + +### ASYNC USAGE ### +# response = await arun_thread(custom_llm_provider="openai", thread_id=thread_id, assistant_id=assistant_id) + +print(f"run_thread: {run_thread}") +``` + + + +```yaml +assistant_settings: + custom_llm_provider: azure + litellm_params: + api_key: os.environ/AZURE_API_KEY + api_base: os.environ/AZURE_API_BASE + api_version: os.environ/AZURE_API_VERSION +``` + +```bash +$ litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +**Get the Assistant** + +```bash +curl "http://0.0.0.0:4000/v1/assistants?order=desc&limit=20" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" +``` + +**Create a Thread** + +```bash +curl http://0.0.0.0:4000/v1/threads \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '' +``` + +**Get a Thread** + +```bash +curl http://0.0.0.0:4000/v1/threads/{thread_id} \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" +``` + +**Add Messages to the Thread** + +```bash +curl http://0.0.0.0:4000/v1/threads/{thread_id}/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "role": "user", + "content": "How does AI work? Explain it in simple terms." + }' +``` + +**Run the Assistant on the Thread** + +```bash +curl http://0.0.0.0:4000/v1/threads/thread_abc123/runs \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "assistant_id": "asst_abc123" + }' +``` + + + + +## Streaming + + + + +```python +from litellm import run_thread_stream +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +message = {"role": "user", "content": "Hey, how's it going?"} + +data = {"custom_llm_provider": "openai", "thread_id": _new_thread.id, "assistant_id": assistant_id, **message} + +run = run_thread_stream(**data) +with run as run: + assert isinstance(run, AssistantEventHandler) + for chunk in run: + print(f"chunk: {chunk}") + run.until_done() +``` + + + + +```bash +curl -X POST 'http://0.0.0.0:4000/threads/{thread_id}/runs' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-D '{ + "assistant_id": "asst_6xVZQFFy1Kw87NbnYeNebxTf", + "stream": true +}' +``` + + + + +## [👉 Proxy API Reference](https://litellm-api.up.railway.app/#/assistants) diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md new file mode 100644 index 000000000..51f3bb5ca --- /dev/null +++ b/docs/my-website/docs/batches.md @@ -0,0 +1,124 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Batches API + +Covers Batches, Files + + +## Quick Start + +Call an existing Assistant. + +- Create File for Batch Completion + +- Create Batch Request + +- Retrieve the Specific Batch and File Content + + + + + +**Create File for Batch Completion** + +```python +from litellm +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +file_name = "openai_batch_completions.jsonl" +_current_dir = os.path.dirname(os.path.abspath(__file__)) +file_path = os.path.join(_current_dir, file_name) +file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), + purpose="batch", + custom_llm_provider="openai", +) +print("Response from creating file=", file_obj) +``` + +**Create Batch Request** + +```python +from litellm +import os + +create_batch_response = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=batch_input_file_id, + custom_llm_provider="openai", + metadata={"key1": "value1", "key2": "value2"}, +) + +print("response from litellm.create_batch=", create_batch_response) +``` + +**Retrieve the Specific Batch and File Content** + +```python + +retrieved_batch = await litellm.aretrieve_batch( + batch_id=create_batch_response.id, custom_llm_provider="openai" +) +print("retrieved batch=", retrieved_batch) +# just assert that we retrieved a non None batch + +assert retrieved_batch.id == create_batch_response.id + +# try to get file content for our original file + +file_content = await litellm.afile_content( + file_id=batch_input_file_id, custom_llm_provider="openai" +) + +print("file content = ", file_content) +``` + + + + +```bash +$ export OPENAI_API_KEY="sk-..." + +$ litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +**Create File for Batch Completion** + +```shell +curl https://api.openai.com/v1/files \ + -H "Authorization: Bearer sk-1234" \ + -F purpose="batch" \ + -F file="@mydata.jsonl" +``` + +**Create Batch Request** + +```bash +curl http://localhost:4000/v1/batches \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' +``` + +**Retrieve the Specific Batch** + +```bash +curl http://localhost:4000/v1/batches/batch_abc123 \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ +``` + + + + +## [👉 Proxy API Reference](https://litellm-api.up.railway.app/#/batch) diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index eb309f9b8..1b8bbd8e0 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -212,6 +212,94 @@ If you run the code two times, response1 will use the cache from the first run t + + +## Switch Cache On / Off Per LiteLLM Call + +LiteLLM supports 4 cache-controls: + +- `no-cache`: *Optional(bool)* When `True`, Will not return a cached response, but instead call the actual endpoint. +- `no-store`: *Optional(bool)* When `True`, Will not cache the response. +- `ttl`: *Optional(int)* - Will cache the response for the user-defined amount of time (in seconds). +- `s-maxage`: *Optional(int)* Will only accept cached responses that are within user-defined range (in seconds). + +[Let us know if you need more](https://github.com/BerriAI/litellm/issues/1218) + + + +Example usage `no-cache` - When `True`, Will not return a cached response + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"no-cache": True}, + ) +``` + + + + + +Example usage `no-store` - When `True`, Will not cache the response. + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"no-store": True}, + ) +``` + + + + +Example usage `ttl` - cache the response for 10 seconds + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"ttl": 10}, + ) +``` + + + + +Example usage `s-maxage` - Will only accept cached responses for 60 seconds + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "hello who are you" + } + ], + cache={"s-maxage": 60}, + ) +``` + + + + ## Cache Context Manager - Enable, Disable, Update Cache diff --git a/docs/my-website/docs/completion/batching.md b/docs/my-website/docs/completion/batching.md index 09f59f743..5854f4db8 100644 --- a/docs/my-website/docs/completion/batching.md +++ b/docs/my-website/docs/completion/batching.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Batching Completion() LiteLLM allows you to: * Send many completion calls to 1 model @@ -51,6 +54,9 @@ This makes parallel calls to the specified `models` and returns the first respon Use this to reduce latency + + + ### Example Code ```python import litellm @@ -68,8 +74,93 @@ response = batch_completion_models( print(result) ``` + + + + + +[how to setup proxy config](#example-setup) + +Just pass a comma-separated string of model names and the flag `fastest_response=True`. + + + + +```bash + +curl -X POST 'http://localhost:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "gpt-4o, groq-llama", # 👈 Comma-separated models + "messages": [ + { + "role": "user", + "content": "What's the weather like in Boston today?" + } + ], + "stream": true, + "fastest_response": true # 👈 FLAG +} + +' +``` + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-4o, groq-llama", # 👈 Comma-separated models + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={"fastest_response": true} # 👈 FLAG +) + +print(response) +``` + + + + +--- + +### Example Setup: + +```yaml +model_list: +- model_name: groq-llama + litellm_params: + model: groq/llama3-8b-8192 + api_key: os.environ/GROQ_API_KEY +- model_name: gpt-4o + litellm_params: + model: gpt-4o + api_key: os.environ/OPENAI_API_KEY +``` + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + + + + ### Output -Returns the first response +Returns the first response in OpenAI format. Cancels other LLM API calls. ```json { "object": "chat.completion", @@ -95,6 +186,7 @@ Returns the first response } ``` + ## Send 1 completion call to many models: Return All Responses This makes parallel calls to the specified models and returns all responses diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index e844c541c..6ad412af8 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -39,38 +39,34 @@ This is a list of openai params we translate across providers. Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider -| Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--| -|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ -|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | -|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ | +| Provider | temperature | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | +|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | +|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ | |Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|Anyscale | ✅ | ✅ | ✅ | ✅ | +|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | |Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ | | | | | |AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -|VertexAI| ✅ | ✅ | | ✅ | | | | | | | | | | | ✅ | | | +|VertexAI| ✅ | ✅ | | ✅ | ✅ | | | | | | | | | | ✅ | | | |Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (for anthropic) | | -|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | -|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | -|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | -|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|Petals| ✅ | ✅ | | ✅ | | | | | | | +|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | +|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | +|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|Petals| ✅ | ✅ | | ✅ | ✅ | | | | | | |Ollama| ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | ✅ | | | |Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | | -|ClarifAI| ✅ | ✅ | | | | | | | | | | | | | | - +|ClarifAI| ✅ | ✅ | |✅ | ✅ | | | | | | | | | | | :::note By default, LiteLLM raises an exception if the openai param being passed in isn't supported. -To drop the param instead, set `litellm.drop_params = True`. +To drop the param instead, set `litellm.drop_params = True` or `completion(..drop_params=True)`. -**For function calling:** - -Add to prompt for non-openai models, set: `litellm.add_function_to_prompt = True`. ::: ## Input Params diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 3dc4cb0e2..0edf937ed 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -10,13 +10,17 @@ For companies that need SSO, user management and professional support for LiteLL This covers: - ✅ **Features under the [LiteLLM Commercial License (Content Mod, Custom Tags, etc.)](https://docs.litellm.ai/docs/proxy/enterprise)** - ✅ [**Secure UI access with Single Sign-On**](../docs/proxy/ui.md#setup-ssoauth-for-ui) +- ✅ [**Audit Logs with retention policy**](../docs/proxy/enterprise.md#audit-logs) - ✅ [**JWT-Auth**](../docs/proxy/token_auth.md) - ✅ [**Prompt Injection Detection**](#prompt-injection-detection-lakeraai) - ✅ [**Invite Team Members to access `/spend` Routes**](../docs/proxy/cost_tracking#allowing-non-proxy-admins-to-access-spend-endpoints) - ✅ **Feature Prioritization** - ✅ **Custom Integrations** - ✅ **Professional Support - Dedicated discord + slack** -- ✅ **Custom SLAs** +- ✅ [**Custom Swagger**](../docs/proxy/enterprise.md#swagger-docs---custom-routes--branding) +- ✅ [**Public Model Hub**](../docs/proxy/enterprise.md#public-model-hub) +- ✅ [**Custom Email Branding**](../docs/proxy/email.md#customizing-email-branding) + ## [COMING SOON] AWS Marketplace Support @@ -33,7 +37,11 @@ Includes all enterprise features. Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them. -We offer custom SLAs based on your needs and the severity of the issue. The standard SLA is 6 hours for Sev0-Sev1 severity and 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday). +- 1 hour for Sev0 issues +- 6 hours for Sev1 +- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) + +**We can offer custom SLAs** based on your needs and the severity of the issue ### What’s the cost of the Self-Managed Enterprise edition? diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md index 7bb4d2c99..10b5b5e68 100644 --- a/docs/my-website/docs/image_generation.md +++ b/docs/my-website/docs/image_generation.md @@ -51,7 +51,7 @@ print(f"response: {response}") - `api_base`: *string (optional)* - The api endpoint you want to call the model with -- `api_version`: *string (optional)* - (Azure-specific) the api version for the call +- `api_version`: *string (optional)* - (Azure-specific) the api version for the call; required for dall-e-3 on Azure - `api_key`: *string (optional)* - The API key to authenticate and authorize requests. If not provided, the default API key is used. @@ -166,4 +166,4 @@ response = litellm.image_generation( vertex_ai_location="us-central1", ) print(f"response: {response}") -``` \ No newline at end of file +``` diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md index 316822227..373b4a96c 100644 --- a/docs/my-website/docs/observability/custom_callback.md +++ b/docs/my-website/docs/observability/custom_callback.md @@ -38,7 +38,7 @@ class MyCustomHandler(CustomLogger): print(f"On Async Success") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Async Success") + print(f"On Async Failure") customHandler = MyCustomHandler() diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 6dd5377ea..07970f599 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -144,6 +144,26 @@ print(response) ``` +You can also pass `metadata` as part of the request header with a `langfuse_*` prefix: + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'langfuse_trace_id: trace-id22' \ + --header 'langfuse_trace_user_id: user-id2' \ + --header 'langfuse_trace_metadata: {"key":"value"}' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + ### Trace & Generation Parameters #### Trace Specific Parameters diff --git a/docs/my-website/docs/projects/llm_cord.md b/docs/my-website/docs/projects/llm_cord.md new file mode 100644 index 000000000..6a28d5c88 --- /dev/null +++ b/docs/my-website/docs/projects/llm_cord.md @@ -0,0 +1,5 @@ +# llmcord.py + +llmcord.py lets you and your friends chat with LLMs directly in your Discord server. It works with practically any LLM, remote or locally hosted. + +Github: https://github.com/jakobdylanc/discord-llm-chatbot diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 38be0c433..ff7fa0483 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -9,6 +9,12 @@ LiteLLM supports - `claude-2.1` - `claude-instant-1.2` +:::info + +Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed + +::: + ## API Keys ```python diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 608bc9d1f..ffe258e97 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -361,47 +361,6 @@ response = completion( ) ``` -### Passing an external BedrockRuntime.Client as a parameter - Completion() -Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth. - -Create a client from session credentials: -```python -import boto3 -from litellm import completion - -bedrock = boto3.client( - service_name="bedrock-runtime", - region_name="us-east-1", - aws_access_key_id="", - aws_secret_access_key="", - aws_session_token="", -) - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_bedrock_client=bedrock, -) -``` - -Create a client from AWS profile in `~/.aws/config`: -```python -import boto3 -from litellm import completion - -dev_session = boto3.Session(profile_name="dev-profile") -bedrock = dev_session.client( - service_name="bedrock-runtime", - region_name="us-east-1", -) - -response = completion( - model="bedrock/anthropic.claude-instant-v1", - messages=[{ "content": "Hello, how are you?","role": "user"}], - aws_bedrock_client=bedrock, -) -``` - ### SSO Login (AWS Profile) - Set `AWS_PROFILE` environment variable - Make bedrock completion call @@ -464,6 +423,56 @@ response = completion( ) ``` + +### Passing an external BedrockRuntime.Client as a parameter - Completion() + +:::warning + +This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues) + +::: + +Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth. + +Create a client from session credentials: +```python +import boto3 +from litellm import completion + +bedrock = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id="", + aws_secret_access_key="", + aws_session_token="", +) + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_bedrock_client=bedrock, +) +``` + +Create a client from AWS profile in `~/.aws/config`: +```python +import boto3 +from litellm import completion + +dev_session = boto3.Session(profile_name="dev-profile") +bedrock = dev_session.client( + service_name="bedrock-runtime", + region_name="us-east-1", +) + +response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + aws_bedrock_client=bedrock, +) +``` + + ## Provisioned throughput models To use provisioned throughput Bedrock models pass - `model=bedrock/`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models) diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index da453c3ce..857aae5bd 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -46,13 +46,13 @@ for chunk in response: ## Supported Models - ALL Groq Models Supported! We support ALL Groq models, just set `groq/` as a prefix when sending completion requests -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` | -| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` | -| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` | +| Model Name | Function Call | +|--------------------|---------------------------------------------------------| +| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` | +| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` | +| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` | | mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` | -| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | +| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | ## Groq - Tool / Function Calling Example diff --git a/docs/my-website/docs/providers/mistral.md b/docs/my-website/docs/providers/mistral.md index 9d13fd017..d9616a522 100644 --- a/docs/my-website/docs/providers/mistral.md +++ b/docs/my-website/docs/providers/mistral.md @@ -42,7 +42,7 @@ for chunk in response: ## Supported Models -All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/c1b25538277206b9f00de5254d80d6a83bb19a29/model_prices_and_context_window.json). +All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). | Model Name | Function Call | |----------------|--------------------------------------------------------------| @@ -52,6 +52,7 @@ All models listed here https://docs.mistral.ai/platform/endpoints are supported. | Mistral 7B | `completion(model="mistral/open-mistral-7b", messages)` | | Mixtral 8x7B | `completion(model="mistral/open-mixtral-8x7b", messages)` | | Mixtral 8x22B | `completion(model="mistral/open-mixtral-8x22b", messages)` | +| Codestral | `completion(model="mistral/codestral-latest", messages)` | ## Function Calling diff --git a/docs/my-website/docs/providers/togetherai.md b/docs/my-website/docs/providers/togetherai.md index d718619f0..1021f5ba8 100644 --- a/docs/my-website/docs/providers/togetherai.md +++ b/docs/my-website/docs/providers/togetherai.md @@ -26,52 +26,52 @@ Example TogetherAI Usage - Note: liteLLM supports all models deployed on Togethe ### Llama LLMs - Chat -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| togethercomputer/llama-2-70b-chat | `completion('together_ai/togethercomputer/llama-2-70b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|-----------------------------------|-------------------------------------------------------------------------|------------------------------------| +| togethercomputer/llama-2-70b-chat | `completion('together_ai/togethercomputer/llama-2-70b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ### Llama LLMs - Language / Instruct -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| togethercomputer/llama-2-70b | `completion('together_ai/togethercomputer/llama-2-70b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/LLaMA-2-7B-32K | `completion('together_ai/togethercomputer/LLaMA-2-7B-32K', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/Llama-2-7B-32K-Instruct | `completion('together_ai/togethercomputer/Llama-2-7B-32K-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/llama-2-7b | `completion('together_ai/togethercomputer/llama-2-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|------------------------------------------|--------------------------------------------------------------------------------|------------------------------------| +| togethercomputer/llama-2-70b | `completion('together_ai/togethercomputer/llama-2-70b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| togethercomputer/LLaMA-2-7B-32K | `completion('together_ai/togethercomputer/LLaMA-2-7B-32K', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| togethercomputer/Llama-2-7B-32K-Instruct | `completion('together_ai/togethercomputer/Llama-2-7B-32K-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| togethercomputer/llama-2-7b | `completion('together_ai/togethercomputer/llama-2-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ### Falcon LLMs -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| togethercomputer/falcon-40b-instruct | `completion('together_ai/togethercomputer/falcon-40b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/falcon-7b-instruct | `completion('together_ai/togethercomputer/falcon-7b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|--------------------------------------|----------------------------------------------------------------------------|------------------------------------| +| togethercomputer/falcon-40b-instruct | `completion('together_ai/togethercomputer/falcon-40b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| togethercomputer/falcon-7b-instruct | `completion('together_ai/togethercomputer/falcon-7b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ### Alpaca LLMs -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| togethercomputer/alpaca-7b | `completion('together_ai/togethercomputer/alpaca-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|----------------------------|------------------------------------------------------------------|------------------------------------| +| togethercomputer/alpaca-7b | `completion('together_ai/togethercomputer/alpaca-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ### Other Chat LLMs -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| HuggingFaceH4/starchat-alpha | `completion('together_ai/HuggingFaceH4/starchat-alpha', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|------------------------------|--------------------------------------------------------------------|------------------------------------| +| HuggingFaceH4/starchat-alpha | `completion('together_ai/HuggingFaceH4/starchat-alpha', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ### Code LLMs -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| togethercomputer/CodeLlama-34b | `completion('together_ai/togethercomputer/CodeLlama-34b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/CodeLlama-34b-Instruct | `completion('together_ai/togethercomputer/CodeLlama-34b-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| togethercomputer/CodeLlama-34b-Python | `completion('together_ai/togethercomputer/CodeLlama-34b-Python', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| defog/sqlcoder | `completion('together_ai/defog/sqlcoder', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| NumbersStation/nsql-llama-2-7B | `completion('together_ai/NumbersStation/nsql-llama-2-7B', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| WizardLM/WizardCoder-15B-V1.0 | `completion('together_ai/WizardLM/WizardCoder-15B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| WizardLM/WizardCoder-Python-34B-V1.0 | `completion('together_ai/WizardLM/WizardCoder-Python-34B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|-----------------------------------------|-------------------------------------------------------------------------------|------------------------------------| +| togethercomputer/CodeLlama-34b | `completion('together_ai/togethercomputer/CodeLlama-34b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| togethercomputer/CodeLlama-34b-Instruct | `completion('together_ai/togethercomputer/CodeLlama-34b-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| togethercomputer/CodeLlama-34b-Python | `completion('together_ai/togethercomputer/CodeLlama-34b-Python', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| defog/sqlcoder | `completion('together_ai/defog/sqlcoder', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| NumbersStation/nsql-llama-2-7B | `completion('together_ai/NumbersStation/nsql-llama-2-7B', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| WizardLM/WizardCoder-15B-V1.0 | `completion('together_ai/WizardLM/WizardCoder-15B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| WizardLM/WizardCoder-Python-34B-V1.0 | `completion('together_ai/WizardLM/WizardCoder-Python-34B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ### Language LLMs -| Model Name | Function Call | Required OS Variables | -|-----------------------------------|------------------------------------------------------------------------|---------------------------------| -| NousResearch/Nous-Hermes-Llama2-13b | `completion('together_ai/NousResearch/Nous-Hermes-Llama2-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| Austism/chronos-hermes-13b | `completion('together_ai/Austism/chronos-hermes-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| upstage/SOLAR-0-70b-16bit | `completion('together_ai/upstage/SOLAR-0-70b-16bit', messages)` | `os.environ['TOGETHERAI_API_KEY']` | -| WizardLM/WizardLM-70B-V1.0 | `completion('together_ai/WizardLM/WizardLM-70B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Model Name | Function Call | Required OS Variables | +|-------------------------------------|---------------------------------------------------------------------------|------------------------------------| +| NousResearch/Nous-Hermes-Llama2-13b | `completion('together_ai/NousResearch/Nous-Hermes-Llama2-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| Austism/chronos-hermes-13b | `completion('together_ai/Austism/chronos-hermes-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| upstage/SOLAR-0-70b-16bit | `completion('together_ai/upstage/SOLAR-0-70b-16bit', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| WizardLM/WizardLM-70B-V1.0 | `completion('together_ai/WizardLM/WizardLM-70B-V1.0', messages)` | `os.environ['TOGETHERAI_API_KEY']` | ## Prompt Templates diff --git a/docs/my-website/docs/providers/vllm.md b/docs/my-website/docs/providers/vllm.md index c22cd4fc2..61dd1fffd 100644 --- a/docs/my-website/docs/providers/vllm.md +++ b/docs/my-website/docs/providers/vllm.md @@ -155,14 +155,14 @@ def default_pt(messages): #### Models we already have Prompt Templates for -| Model Name | Works for Models | Function Call | -| -------- | -------- | -------- | -| meta-llama/Llama-2-7b-chat | All meta-llama llama2 chat models| `completion(model='vllm/meta-llama/Llama-2-7b', messages=messages, api_base="your_api_endpoint")` | -| tiiuae/falcon-7b-instruct | All falcon instruct models | `completion(model='vllm/tiiuae/falcon-7b-instruct', messages=messages, api_base="your_api_endpoint")` | -| mosaicml/mpt-7b-chat | All mpt chat models | `completion(model='vllm/mosaicml/mpt-7b-chat', messages=messages, api_base="your_api_endpoint")` | -| codellama/CodeLlama-34b-Instruct-hf | All codellama instruct models | `completion(model='vllm/codellama/CodeLlama-34b-Instruct-hf', messages=messages, api_base="your_api_endpoint")` | -| WizardLM/WizardCoder-Python-34B-V1.0 | All wizardcoder models | `completion(model='vllm/WizardLM/WizardCoder-Python-34B-V1.0', messages=messages, api_base="your_api_endpoint")` | -| Phind/Phind-CodeLlama-34B-v2 | All phind-codellama models | `completion(model='vllm/Phind/Phind-CodeLlama-34B-v2', messages=messages, api_base="your_api_endpoint")` | +| Model Name | Works for Models | Function Call | +|--------------------------------------|-----------------------------------|------------------------------------------------------------------------------------------------------------------| +| meta-llama/Llama-2-7b-chat | All meta-llama llama2 chat models | `completion(model='vllm/meta-llama/Llama-2-7b', messages=messages, api_base="your_api_endpoint")` | +| tiiuae/falcon-7b-instruct | All falcon instruct models | `completion(model='vllm/tiiuae/falcon-7b-instruct', messages=messages, api_base="your_api_endpoint")` | +| mosaicml/mpt-7b-chat | All mpt chat models | `completion(model='vllm/mosaicml/mpt-7b-chat', messages=messages, api_base="your_api_endpoint")` | +| codellama/CodeLlama-34b-Instruct-hf | All codellama instruct models | `completion(model='vllm/codellama/CodeLlama-34b-Instruct-hf', messages=messages, api_base="your_api_endpoint")` | +| WizardLM/WizardCoder-Python-34B-V1.0 | All wizardcoder models | `completion(model='vllm/WizardLM/WizardCoder-Python-34B-V1.0', messages=messages, api_base="your_api_endpoint")` | +| Phind/Phind-CodeLlama-34B-v2 | All phind-codellama models | `completion(model='vllm/Phind/Phind-CodeLlama-34B-v2', messages=messages, api_base="your_api_endpoint")` | #### Custom prompt templates diff --git a/docs/my-website/docs/providers/watsonx.md b/docs/my-website/docs/providers/watsonx.md index d8c5740a8..7a42a54ed 100644 --- a/docs/my-website/docs/providers/watsonx.md +++ b/docs/my-website/docs/providers/watsonx.md @@ -251,23 +251,23 @@ response = completion( Here are some examples of models available in IBM watsonx.ai that you can use with LiteLLM: -| Mode Name | Command | -| ---------- | --------- | -| Flan T5 XXL | `completion(model=watsonx/google/flan-t5-xxl, messages=messages)` | -| Flan Ul2 | `completion(model=watsonx/google/flan-ul2, messages=messages)` | -| Mt0 XXL | `completion(model=watsonx/bigscience/mt0-xxl, messages=messages)` | -| Gpt Neox | `completion(model=watsonx/eleutherai/gpt-neox-20b, messages=messages)` | -| Mpt 7B Instruct2 | `completion(model=watsonx/ibm/mpt-7b-instruct2, messages=messages)` | -| Starcoder | `completion(model=watsonx/bigcode/starcoder, messages=messages)` | -| Llama 2 70B Chat | `completion(model=watsonx/meta-llama/llama-2-70b-chat, messages=messages)` | -| Llama 2 13B Chat | `completion(model=watsonx/meta-llama/llama-2-13b-chat, messages=messages)` | -| Granite 13B Instruct | `completion(model=watsonx/ibm/granite-13b-instruct-v1, messages=messages)` | -| Granite 13B Chat | `completion(model=watsonx/ibm/granite-13b-chat-v1, messages=messages)` | -| Flan T5 XL | `completion(model=watsonx/google/flan-t5-xl, messages=messages)` | -| Granite 13B Chat V2 | `completion(model=watsonx/ibm/granite-13b-chat-v2, messages=messages)` | -| Granite 13B Instruct V2 | `completion(model=watsonx/ibm/granite-13b-instruct-v2, messages=messages)` | -| Elyza Japanese Llama 2 7B Instruct | `completion(model=watsonx/elyza/elyza-japanese-llama-2-7b-instruct, messages=messages)` | -| Mixtral 8X7B Instruct V01 Q | `completion(model=watsonx/ibm-mistralai/mixtral-8x7b-instruct-v01-q, messages=messages)` | +| Mode Name | Command | +|------------------------------------|------------------------------------------------------------------------------------------| +| Flan T5 XXL | `completion(model=watsonx/google/flan-t5-xxl, messages=messages)` | +| Flan Ul2 | `completion(model=watsonx/google/flan-ul2, messages=messages)` | +| Mt0 XXL | `completion(model=watsonx/bigscience/mt0-xxl, messages=messages)` | +| Gpt Neox | `completion(model=watsonx/eleutherai/gpt-neox-20b, messages=messages)` | +| Mpt 7B Instruct2 | `completion(model=watsonx/ibm/mpt-7b-instruct2, messages=messages)` | +| Starcoder | `completion(model=watsonx/bigcode/starcoder, messages=messages)` | +| Llama 2 70B Chat | `completion(model=watsonx/meta-llama/llama-2-70b-chat, messages=messages)` | +| Llama 2 13B Chat | `completion(model=watsonx/meta-llama/llama-2-13b-chat, messages=messages)` | +| Granite 13B Instruct | `completion(model=watsonx/ibm/granite-13b-instruct-v1, messages=messages)` | +| Granite 13B Chat | `completion(model=watsonx/ibm/granite-13b-chat-v1, messages=messages)` | +| Flan T5 XL | `completion(model=watsonx/google/flan-t5-xl, messages=messages)` | +| Granite 13B Chat V2 | `completion(model=watsonx/ibm/granite-13b-chat-v2, messages=messages)` | +| Granite 13B Instruct V2 | `completion(model=watsonx/ibm/granite-13b-instruct-v2, messages=messages)` | +| Elyza Japanese Llama 2 7B Instruct | `completion(model=watsonx/elyza/elyza-japanese-llama-2-7b-instruct, messages=messages)` | +| Mixtral 8X7B Instruct V01 Q | `completion(model=watsonx/ibm-mistralai/mixtral-8x7b-instruct-v01-q, messages=messages)` | For a list of all available models in watsonx.ai, see [here](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx&locale=en&audience=wdp). @@ -275,10 +275,10 @@ For a list of all available models in watsonx.ai, see [here](https://dataplatfor ## Supported IBM watsonx.ai Embedding Models -| Model Name | Function Call | -|----------------------|---------------------------------------------| -| Slate 30m | `embedding(model="watsonx/ibm/slate-30m-english-rtrvr", input=input)` | -| Slate 125m | `embedding(model="watsonx/ibm/slate-125m-english-rtrvr", input=input)` | +| Model Name | Function Call | +|------------|------------------------------------------------------------------------| +| Slate 30m | `embedding(model="watsonx/ibm/slate-30m-english-rtrvr", input=input)` | +| Slate 125m | `embedding(model="watsonx/ibm/slate-125m-english-rtrvr", input=input)` | For a list of all available embedding models in watsonx.ai, see [here](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). \ No newline at end of file diff --git a/docs/my-website/docs/providers/xinference.md b/docs/my-website/docs/providers/xinference.md index 3c927dcb4..3686c0209 100644 --- a/docs/my-website/docs/providers/xinference.md +++ b/docs/my-website/docs/providers/xinference.md @@ -37,26 +37,26 @@ print(response) ## Supported Models All models listed here https://inference.readthedocs.io/en/latest/models/builtin/embedding/index.html are supported -| Model Name | Function Call | -|------------------------------|--------------------------------------------------------| -| bge-base-en | `embedding(model="xinference/bge-base-en", input)` | -| bge-base-en-v1.5 | `embedding(model="xinference/bge-base-en-v1.5", input)` | -| bge-base-zh | `embedding(model="xinference/bge-base-zh", input)` | -| bge-base-zh-v1.5 | `embedding(model="xinference/bge-base-zh-v1.5", input)` | -| bge-large-en | `embedding(model="xinference/bge-large-en", input)` | -| bge-large-en-v1.5 | `embedding(model="xinference/bge-large-en-v1.5", input)` | -| bge-large-zh | `embedding(model="xinference/bge-large-zh", input)` | -| bge-large-zh-noinstruct | `embedding(model="xinference/bge-large-zh-noinstruct", input)` | -| bge-large-zh-v1.5 | `embedding(model="xinference/bge-large-zh-v1.5", input)` | -| bge-small-en-v1.5 | `embedding(model="xinference/bge-small-en-v1.5", input)` | -| bge-small-zh | `embedding(model="xinference/bge-small-zh", input)` | -| bge-small-zh-v1.5 | `embedding(model="xinference/bge-small-zh-v1.5", input)` | -| e5-large-v2 | `embedding(model="xinference/e5-large-v2", input)` | -| gte-base | `embedding(model="xinference/gte-base", input)` | -| gte-large | `embedding(model="xinference/gte-large", input)` | -| jina-embeddings-v2-base-en | `embedding(model="xinference/jina-embeddings-v2-base-en", input)` | -| jina-embeddings-v2-small-en | `embedding(model="xinference/jina-embeddings-v2-small-en", input)` | -| multilingual-e5-large | `embedding(model="xinference/multilingual-e5-large", input)` | +| Model Name | Function Call | +|-----------------------------|--------------------------------------------------------------------| +| bge-base-en | `embedding(model="xinference/bge-base-en", input)` | +| bge-base-en-v1.5 | `embedding(model="xinference/bge-base-en-v1.5", input)` | +| bge-base-zh | `embedding(model="xinference/bge-base-zh", input)` | +| bge-base-zh-v1.5 | `embedding(model="xinference/bge-base-zh-v1.5", input)` | +| bge-large-en | `embedding(model="xinference/bge-large-en", input)` | +| bge-large-en-v1.5 | `embedding(model="xinference/bge-large-en-v1.5", input)` | +| bge-large-zh | `embedding(model="xinference/bge-large-zh", input)` | +| bge-large-zh-noinstruct | `embedding(model="xinference/bge-large-zh-noinstruct", input)` | +| bge-large-zh-v1.5 | `embedding(model="xinference/bge-large-zh-v1.5", input)` | +| bge-small-en-v1.5 | `embedding(model="xinference/bge-small-en-v1.5", input)` | +| bge-small-zh | `embedding(model="xinference/bge-small-zh", input)` | +| bge-small-zh-v1.5 | `embedding(model="xinference/bge-small-zh-v1.5", input)` | +| e5-large-v2 | `embedding(model="xinference/e5-large-v2", input)` | +| gte-base | `embedding(model="xinference/gte-base", input)` | +| gte-large | `embedding(model="xinference/gte-large", input)` | +| jina-embeddings-v2-base-en | `embedding(model="xinference/jina-embeddings-v2-base-en", input)` | +| jina-embeddings-v2-small-en | `embedding(model="xinference/jina-embeddings-v2-small-en", input)` | +| multilingual-e5-large | `embedding(model="xinference/multilingual-e5-large", input)` | diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 9574a4dc3..402de410c 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -62,6 +62,23 @@ curl -X GET 'http://localhost:4000/health/services?service=slack' \ -H 'Authorization: Bearer sk-1234' ``` +## Advanced - Redacting Messages from Alerts + +By default alerts show the `messages/input` passed to the LLM. If you want to redact this from slack alerting set the following setting on your config + + +```shell +general_settings: + alerting: ["slack"] + alert_types: ["spend_reports"] + +litellm_settings: + redact_messages_in_exceptions: True +``` + + + + ## Advanced - Opting into specific alert types Set `alert_types` if you want to Opt into only specific alert types @@ -178,23 +195,26 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \ } ``` -**API Spec for Webhook Event** +## **API Spec for Webhook Event** - `spend` *float*: The current spend amount for the 'event_group'. -- `max_budget` *float*: The maximum allowed budget for the 'event_group'. +- `max_budget` *float or null*: The maximum allowed budget for the 'event_group'. null if not set. - `token` *str*: A hashed value of the key, used for authentication or identification purposes. -- `user_id` *str or null*: The ID of the user associated with the event (optional). +- `customer_id` *str or null*: The ID of the customer associated with the event (optional). +- `internal_user_id` *str or null*: The ID of the internal user associated with the event (optional). - `team_id` *str or null*: The ID of the team associated with the event (optional). -- `user_email` *str or null*: The email of the user associated with the event (optional). +- `user_email` *str or null*: The email of the internal user associated with the event (optional). - `key_alias` *str or null*: An alias for the key associated with the event (optional). - `projected_exceeded_date` *str or null*: The date when the budget is projected to be exceeded, returned when 'soft_budget' is set for key (optional). - `projected_spend` *float or null*: The projected spend amount, returned when 'soft_budget' is set for key (optional). - `event` *Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"]*: The type of event that triggered the webhook. Possible values are: + * "spend_tracked": Emitted whenver spend is tracked for a customer id. * "budget_crossed": Indicates that the spend has exceeded the max budget. * "threshold_crossed": Indicates that spend has crossed a threshold (currently sent when 85% and 95% of budget is reached). * "projected_limit_exceeded": For "key" only - Indicates that the projected spend is expected to exceed the soft budget threshold. -- `event_group` *Literal["user", "key", "team", "proxy"]*: The group associated with the event. Possible values are: - * "user": The event is related to a specific user. +- `event_group` *Literal["customer", "internal_user", "key", "team", "proxy"]*: The group associated with the event. Possible values are: + * "customer": The event is related to a specific customer + * "internal_user": The event is related to a specific internal user. * "key": The event is related to a specific key. * "team": The event is related to a team. * "proxy": The event is related to a proxy. diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index 15b1921b0..25fb4ce34 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -283,7 +283,7 @@ litellm_settings: ### Turn on / off caching per request. -The proxy support 3 cache-controls: +The proxy support 4 cache-controls: - `ttl`: *Optional(int)* - Will cache the response for the user-defined amount of time (in seconds). - `s-maxage`: *Optional(int)* Will only accept cached responses that are within user-defined range (in seconds). @@ -374,6 +374,33 @@ chat_completion = client.chat.completions.create( ) ``` +### Turn on / off caching per Key. + +1. Add cache params when creating a key [full list](#turn-on--off-caching-per-key) + +```bash +curl -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-D '{ + "user_id": "222", + "metadata": { + "cache": { + "no-cache": true + } + } +}' +``` + +2. Test it! + +```bash +curl -X POST 'http://localhost:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer ' \ +-D '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "bom dia"}]}' +``` + ### Deleting Cache Keys - `/cache/delete` In order to delete a cache key, send a request to `/cache/delete` with the `keys` you want to delete diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 5eeb05f36..2552c2004 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -80,6 +80,13 @@ For more provider-specific info, [go here](../providers/) $ litellm --config /path/to/config.yaml ``` +:::tip + +Run with `--detailed_debug` if you need detailed debug logs + +```shell +$ litellm --config /path/to/config.yaml --detailed_debug +::: ### Using Proxy - Curl Request, OpenAI Package, Langchain, Langchain JS Calling a model group diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index 7405fd123..b63fab106 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -1,22 +1,163 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; # 💸 Spend Tracking Track spend for keys, users, and teams across 100+ LLMs. -## Getting Spend Reports - To Charge Other Teams, API Keys +### How to Track Spend with LiteLLM -Use the `/global/spend/report` endpoint to get daily spend per team, with a breakdown of spend per API Key, Model +**Step 1** -### Example Request +👉 [Setup LiteLLM with a Database](https://docs.litellm.ai/docs/proxy/deploy) + + +**Step2** Send `/chat/completions` request + + + + + + +```python +import openai +client = openai.OpenAI( + api_key="sk-1234", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="llama3", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + user="palantir", + extra_body={ + "metadata": { + "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"] + } + } +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body ```shell -curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30' \ +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --data '{ + "model": "llama3", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "user": "palantir", + "metadata": { + "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"] + } +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "sk-1234" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "llama3", + user="palantir", + extra_body={ + "metadata": { + "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"] + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + +**Step3 - Verify Spend Tracked** +That's IT. Now Verify your spend was tracked + +The following spend gets tracked in Table `LiteLLM_SpendLogs` + +```json +{ + "api_key": "fe6b0cab4ff5a5a8df823196cc8a450*****", # Hash of API Key used + "user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`. + "team_id": "e8d1460f-846c-45d7-9b43-55f3cc52ac32", # Team (LiteLLM_TeamTable) that owns `api_key=sk-1234` + "request_tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"],# Tags sent in request + "end_user": "palantir", # Customer - the `user` sent in the request + "model_group": "llama3", # "model" passed to LiteLLM + "api_base": "https://api.groq.com/openai/v1/", # "api_base" of model used by LiteLLM + "spend": 0.000002, # Spend in $ + "total_tokens": 100, + "completion_tokens": 80, + "prompt_tokens": 20, + +} +``` + +Navigate to the Usage Tab on the LiteLLM UI (found on https://your-proxy-endpoint/ui) and verify you see spend tracked under `Usage` + + + +## API Endpoints to get Spend +#### Getting Spend Reports - To Charge Other Teams, Customers + +Use the `/global/spend/report` endpoint to get daily spend report per +- team +- customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm) + + + + + +##### Example Request + +👉 Key Change: Specify `group_by=team` + +```shell +curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=team' \ -H 'Authorization: Bearer sk-1234' ``` -### Example Response +##### Example Response @@ -125,7 +266,70 @@ Output from script -## Allowing Non-Proxy Admins to access `/spend` endpoints + + + + + +##### Example Request + +👉 Key Change: Specify `group_by=customer` + + +```shell +curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=customer' \ + -H 'Authorization: Bearer sk-1234' +``` + +##### Example Response + + +```shell +[ + { + "group_by_day": "2024-04-30T00:00:00+00:00", + "customers": [ + { + "customer": "palantir", + "total_spend": 0.0015265, + "metadata": [ # see the spend by unique(key + model) + { + "model": "gpt-4", + "spend": 0.00123, + "total_tokens": 28, + "api_key": "88dc28.." # the hashed api key + }, + { + "model": "gpt-4", + "spend": 0.00123, + "total_tokens": 28, + "api_key": "a73dc2.." # the hashed api key + }, + { + "model": "chatgpt-v-2", + "spend": 0.000214, + "total_tokens": 122, + "api_key": "898c28.." # the hashed api key + }, + { + "model": "gpt-3.5-turbo", + "spend": 0.0000825, + "total_tokens": 85, + "api_key": "84dc28.." # the hashed api key + } + ] + } + ] + } +] +``` + + + + + + +#### Allowing Non-Proxy Admins to access `/spend` endpoints Use this when you want non-proxy admins to access `/spend` endpoints @@ -135,7 +339,7 @@ Schedule a [meeting with us to get your Enterprise License](https://calendly.com ::: -### Create Key +##### Create Key Create Key with with `permissions={"get_spend_routes": true}` ```shell curl --location 'http://0.0.0.0:4000/key/generate' \ @@ -146,7 +350,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ }' ``` -### Use generated key on `/spend` endpoints +##### Use generated key on `/spend` endpoints Access spend Routes with newly generate keys ```shell @@ -156,14 +360,14 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end -## Reset Team, API Key Spend - MASTER KEY ONLY +#### Reset Team, API Key Spend - MASTER KEY ONLY Use `/global/spend/reset` if you want to: - Reset the Spend for all API Keys, Teams. The `spend` for ALL Teams and Keys in `LiteLLM_TeamTable` and `LiteLLM_VerificationToken` will be set to `spend=0` - LiteLLM will maintain all the logs in `LiteLLMSpendLogs` for Auditing Purposes -### Request +##### Request Only the `LITELLM_MASTER_KEY` you set can access this route ```shell curl -X POST \ @@ -172,7 +376,7 @@ curl -X POST \ -H 'Content-Type: application/json' ``` -### Expected Responses +##### Expected Responses ```shell {"message":"Spend for all API Keys and Teams reset successfully","status":"success"} @@ -181,11 +385,11 @@ curl -X POST \ -## Spend Tracking for Azure +## Spend Tracking for Azure OpenAI Models Set base model for cost tracking azure image-gen call -### Image Generation +#### Image Generation ```yaml model_list: @@ -200,7 +404,7 @@ model_list: mode: image_generation ``` -### Chat Completions / Embeddings +#### Chat Completions / Embeddings **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking @@ -219,4 +423,8 @@ model_list: api_version: "2023-07-01-preview" model_info: base_model: azure/gpt-4-1106-preview -``` \ No newline at end of file +``` + +## Custom Input/Output Pricing + +👉 Head to [Custom Input/Output Pricing](https://docs.litellm.ai/docs/proxy/custom_pricing) to setup custom pricing or your models \ No newline at end of file diff --git a/docs/my-website/docs/proxy/customers.md b/docs/my-website/docs/proxy/customers.md new file mode 100644 index 000000000..94000cde2 --- /dev/null +++ b/docs/my-website/docs/proxy/customers.md @@ -0,0 +1,251 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 🙋‍♂️ Customers + +Track spend, set budgets for your customers. + +## Tracking Customer Credit + +### 1. Make LLM API call w/ Customer ID + +Make a /chat/completions call, pass 'user' - First call Works + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY + --data ' { + "model": "azure-gpt-3.5", + "user": "ishaan3", # 👈 CUSTOMER ID + "messages": [ + { + "role": "user", + "content": "what time is it" + } + ] + }' +``` + +The customer_id will be upserted into the DB with the new spend. + +If the customer_id already exists, spend will be incremented. + +### 2. Get Customer Spend + + + + +Call `/customer/info` to get a customer's all up spend + +```bash +curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=ishaan3' \ # 👈 CUSTOMER ID + -H 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY +``` + +Expected Response: + +``` +{ + "user_id": "ishaan3", + "blocked": false, + "alias": null, + "spend": 0.001413, + "allowed_model_region": null, + "default_model": null, + "litellm_budget_table": null +} +``` + + + + +To update spend in your client-side DB, point the proxy to your webhook. + +E.g. if your server is `https://webhook.site` and your listening on `6ab090e8-c55f-4a23-b075-3209f5c57906` + +1. Add webhook url to your proxy environment: + +```bash +export WEBHOOK_URL="https://webhook.site/6ab090e8-c55f-4a23-b075-3209f5c57906" +``` + +2. Add 'webhook' to config.yaml + +```yaml +general_settings: + alerting: ["webhook"] # 👈 KEY CHANGE +``` + +3. Test it! + +```bash +curl -X POST 'http://localhost:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "mistral", + "messages": [ + { + "role": "user", + "content": "What's the weather like in Boston today?" + } + ], + "user": "krrish12" +} +' +``` + +Expected Response + +```json +{ + "spend": 0.0011120000000000001, # 👈 SPEND + "max_budget": null, + "token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b", + "customer_id": "krrish12", # 👈 CUSTOMER ID + "user_id": null, + "team_id": null, + "user_email": null, + "key_alias": null, + "projected_exceeded_date": null, + "projected_spend": null, + "event": "spend_tracked", + "event_group": "customer", + "event_message": "Customer spend tracked. Customer=krrish12, spend=0.0011120000000000001" +} +``` + +[See Webhook Spec](./alerting.md#api-spec-for-webhook-event) + + + + + +## Setting Customer Budgets + +Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy + +### Quick Start + +Create / Update a customer with budget + +**Create New Customer w/ budget** +```bash +curl -X POST 'http://0.0.0.0:4000/customer/new' + -H 'Authorization: Bearer sk-1234' + -H 'Content-Type: application/json' + -D '{ + "user_id" : "my-customer-id", + "max_budget": "0", # 👈 CAN BE FLOAT + }' +``` + +**Test it!** + +```bash +curl -X POST 'http://localhost:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "mistral", + "messages": [ + { + "role": "user", + "content": "What'\''s the weather like in Boston today?" + } + ], + "user": "ishaan-jaff-48" +} +``` + +### Assign Pricing Tiers + +Create and assign customers to pricing tiers. + +#### 1. Create a budget + + + + +- Go to the 'Budgets' tab on the UI. +- Click on '+ Create Budget'. +- Create your pricing tier (e.g. 'my-free-tier' with budget $4). This means each user on this pricing tier will have a max budget of $4. + + + + + + +Use the `/budget/new` endpoint for creating a new budget. [API Reference](https://litellm-api.up.railway.app/#/budget%20management/new_budget_budget_new_post) + +```bash +curl -X POST 'http://localhost:4000/budget/new' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "budget_id": "my-free-tier", + "max_budget": 4 +} +``` + + + + + +#### 2. Assign Budget to Customer + +In your application code, assign budget when creating a new customer. + +Just use the `budget_id` used when creating the budget. In our example, this is `my-free-tier`. + +```bash +curl -X POST 'http://localhost:4000/customer/new' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "user_id": "my-customer-id", + "budget_id": "my-free-tier" # 👈 KEY CHANGE +} +``` + +#### 3. Test it! + + + + +```bash +curl -X POST 'http://localhost:4000/customer/new' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "user_id": "my-customer-id", + "budget_id": "my-free-tier" # 👈 KEY CHANGE +} +``` + + + + +```python +from openai import OpenAI +client = OpenAI( + base_url=" + \ No newline at end of file diff --git a/docs/my-website/docs/proxy/debugging.md b/docs/my-website/docs/proxy/debugging.md index b9f2ba8da..571a97c0e 100644 --- a/docs/my-website/docs/proxy/debugging.md +++ b/docs/my-website/docs/proxy/debugging.md @@ -42,6 +42,14 @@ Set `JSON_LOGS="True"` in your env: ```bash export JSON_LOGS="True" ``` +**OR** + +Set `json_logs: true` in your yaml: + +```yaml +litellm_settings: + json_logs: true +``` Start proxy @@ -49,4 +57,35 @@ Start proxy $ litellm ``` -The proxy will now all logs in json format. \ No newline at end of file +The proxy will now all logs in json format. + +## Control Log Output + +Turn off fastapi's default 'INFO' logs + +1. Turn on 'json logs' +```yaml +litellm_settings: + json_logs: true +``` + +2. Set `LITELLM_LOG` to 'ERROR' + +Only get logs if an error occurs. + +```bash +LITELLM_LOG="ERROR" +``` + +3. Start proxy + + +```bash +$ litellm +``` + +Expected Output: + +```bash +# no info statements +``` \ No newline at end of file diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index f9a7db2d4..b756f56e2 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -7,6 +7,23 @@ You can find the Dockerfile to build litellm proxy [here](https://github.com/Ber ## Quick Start +To start using Litellm, run the following commands in a shell: + +```bash +# Get the code +git clone https://github.com/BerriAI/litellm + +# Go to folder +cd litellm + +# Add the master key +echo 'LITELLM_MASTER_KEY="sk-1234"' > .env +source .env + +# Start +docker-compose up +``` + @@ -243,7 +260,7 @@ Requirements: -We maintain a [seperate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database +We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database ```shell docker pull ghcr.io/berriai/litellm-database:main-latest diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index 62dfeccf7..2551f4359 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -2,12 +2,6 @@ import Image from '@theme/IdealImage'; # ✨ 📧 Email Notifications -:::info - -This is an Enterprise only feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) - -::: - Send an Email to your users when: - A Proxy API Key is created for them - Their API Key crosses it's Budget @@ -38,6 +32,12 @@ That's it ! start your proxy ## Customizing Email Branding +:::info + +Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + LiteLLM allows you to customize the: - Logo on the Email - Email support contact diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index c47589e8a..48cab6093 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -2,29 +2,317 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# ✨ Enterprise Features - Content Mod, SSO, Custom Swagger +# ✨ Enterprise Features - SSO, Audit Logs, Guardrails -Features here are behind a commercial license in our `/enterprise` folder. [**See Code**](https://github.com/BerriAI/litellm/tree/main/enterprise) +:::tip -:::info - -[Get Started with Enterprise here](https://github.com/BerriAI/litellm/tree/main/enterprise) +Get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) ::: Features: - ✅ [SSO for Admin UI](./ui.md#✨-enterprise-features) -- ✅ Content Moderation with LLM Guard, LlamaGuard, Google Text Moderations -- ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection-lakeraai) +- ✅ [Audit Logs](#audit-logs) +- ✅ [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags) +- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests) +- ✅ [Content Moderation with LLM Guard, LlamaGuard, Google Text Moderations](#content-moderation) +- ✅ [Prompt Injection Detection (with LakeraAI API)](#prompt-injection-detection---lakeraai) +- ✅ [Custom Branding + Routes on Swagger Docs](#swagger-docs---custom-routes--branding) - ✅ Reject calls from Blocked User list - ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors) -- ✅ Don't log/store specific requests to Langfuse, Sentry, etc. (eg confidential LLM requests) -- ✅ Tracking Spend for Custom Tags -- ✅ Custom Branding + Routes on Swagger Docs + +## Audit Logs + +Store Audit logs for **Create, Update Delete Operations** done on `Teams` and `Virtual Keys` + +**Step 1** Switch on audit Logs +```shell +litellm_settings: + store_audit_logs: true +``` + +Start the litellm proxy with this config + +**Step 2** Test it - Create a Team + +```shell +curl --location 'http://0.0.0.0:4000/team/new' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "max_budget": 2 + }' +``` + +**Step 3** Expected Log + +```json +{ + "id": "e1760e10-4264-4499-82cd-c08c86c8d05b", + "updated_at": "2024-06-06T02:10:40.836420+00:00", + "changed_by": "109010464461339474872", + "action": "created", + "table_name": "LiteLLM_TeamTable", + "object_id": "82e725b5-053f-459d-9a52-867191635446", + "before_value": null, + "updated_values": { + "team_id": "82e725b5-053f-459d-9a52-867191635446", + "admins": [], + "members": [], + "members_with_roles": [ + { + "role": "admin", + "user_id": "109010464461339474872" + } + ], + "max_budget": 2.0, + "models": [], + "blocked": false + } +} +``` + + +## Tracking Spend for Custom Tags + +Requirements: + +- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys) + +#### Usage - /chat/completions requests with request tags + + + + + + + +Set `extra_body={"metadata": { }}` to `metadata` you want to pass + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] + } + } +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": {"tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"]} +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "metadata": { + "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + +#### Viewing Spend per tag + +#### `/spend/tags` Request Format +```shell +curl -X GET "http://0.0.0.0:4000/spend/tags" \ +-H "Authorization: Bearer sk-1234" +``` + +#### `/spend/tags`Response Format +```shell +[ + { + "individual_request_tag": "model-anthropic-claude-v2.1", + "log_count": 6, + "total_spend": 0.000672 + }, + { + "individual_request_tag": "app-ishaan-local", + "log_count": 4, + "total_spend": 0.000448 + }, + { + "individual_request_tag": "app-ishaan-prod", + "log_count": 2, + "total_spend": 0.000224 + } +] + +``` + + +## Enforce Required Params for LLM Requests +Use this when you want to enforce all requests to include certain params. Example you need all requests to include the `user` and `["metadata]["generation_name"]` params. + +**Step 1** Define all Params you want to enforce on config.yaml + +This means `["user"]` and `["metadata]["generation_name"]` are required in all LLM Requests to LiteLLM + +```yaml +general_settings: + master_key: sk-1234 + enforced_params: + - user + - metadata.generation_name +``` + +Start LiteLLM Proxy + +**Step 2 Verify if this works** + + + + + +```shell +curl --location 'http://localhost:4000/chat/completions' \ + --header 'Authorization: Bearer sk-5fmYeaUEbAMpwBNT-QpxyA' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "hi" + } + ] +}' +``` + +Expected Response + +```shell +{"error":{"message":"Authentication Error, BadRequest please pass param=user in request body. This is a required param","type":"auth_error","param":"None","code":401}}% +``` + + + + + +```shell +curl --location 'http://localhost:4000/chat/completions' \ + --header 'Authorization: Bearer sk-5fmYeaUEbAMpwBNT-QpxyA' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "user": "gm", + "messages": [ + { + "role": "user", + "content": "hi" + } + ], + "metadata": {} +}' +``` + +Expected Response + +```shell +{"error":{"message":"Authentication Error, BadRequest please pass param=[metadata][generation_name] in request body. This is a required param","type":"auth_error","param":"None","code":401}}% +``` + + + + + +```shell +curl --location 'http://localhost:4000/chat/completions' \ + --header 'Authorization: Bearer sk-5fmYeaUEbAMpwBNT-QpxyA' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "user": "gm", + "messages": [ + { + "role": "user", + "content": "hi" + } + ], + "metadata": {"generation_name": "prod-app"} +}' +``` + +Expected Response + +```shell +{"id":"chatcmpl-9XALnHqkCBMBKrOx7Abg0hURHqYtY","choices":[{"finish_reason":"stop","index":0,"message":{"content":"Hello! How can I assist you today?","role":"assistant"}}],"created":1717691639,"model":"gpt-3.5-turbo-0125","object":"chat.completion","system_fingerprint":null,"usage":{"completion_tokens":9,"prompt_tokens":8,"total_tokens":17}}% +``` + + + + + + + ## Content Moderation -### Content Moderation with LLM Guard +#### Content Moderation with LLM Guard Set the LLM Guard API Base in your environment @@ -159,7 +447,7 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \ -### Content Moderation with LlamaGuard +#### Content Moderation with LlamaGuard Currently works with Sagemaker's LlamaGuard endpoint. @@ -193,7 +481,7 @@ callbacks: ["llamaguard_moderations"] -### Content Moderation with Google Text Moderation +#### Content Moderation with Google Text Moderation Requires your GOOGLE_APPLICATION_CREDENTIALS to be set in your .env (same as VertexAI). @@ -249,7 +537,7 @@ Here are the category specific values: -### Content Moderation with OpenAI Moderations +#### Content Moderation with OpenAI Moderations Use this if you want to reject /chat, /completions, /embeddings calls that fail OpenAI Moderations checks @@ -275,7 +563,7 @@ Step 1 Set a `LAKERA_API_KEY` in your env LAKERA_API_KEY="7a91a1a6059da*******" ``` -Step 2. Add `lakera_prompt_injection` to your calbacks +Step 2. Add `lakera_prompt_injection` to your callbacks ```yaml litellm_settings: @@ -301,6 +589,42 @@ curl --location 'http://localhost:4000/chat/completions' \ }' ``` +## Swagger Docs - Custom Routes + Branding + +:::info + +Requires a LiteLLM Enterprise key to use. Get a free 2-week license [here](https://forms.gle/sTDVprBs18M4V8Le8) + +::: + +Set LiteLLM Key in your environment + +```bash +LITELLM_LICENSE="" +``` + +#### Customize Title + Description + +In your environment, set: + +```bash +DOCS_TITLE="TotalGPT" +DOCS_DESCRIPTION="Sample Company Description" +``` + +#### Customize Routes + +Hide admin routes from users. + +In your environment, set: + +```bash +DOCS_FILTERED="True" # only shows openai routes to user +``` + + + + ## Enable Blocked User Lists If any call is made to proxy with this user id, it'll be rejected - use this if you want to let users opt-out of ai features @@ -416,173 +740,9 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ' ``` -## Tracking Spend for Custom Tags -Requirements: +## Public Model Hub -- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys) +Share a public page of available models for users -### Usage - /chat/completions requests with request tags - - - - - - - -Set `extra_body={"metadata": { }}` to `metadata` you want to pass - -```python -import openai -client = openai.OpenAI( - api_key="anything", - base_url="http://0.0.0.0:4000" -) - -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } - ], - extra_body={ - "metadata": { - "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] - } - } -) - -print(response) -``` - - - - -Pass `metadata` as part of the request body - -```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - "metadata": {"tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"]} -}' -``` - - - -```python -from langchain.chat_models import ChatOpenAI -from langchain.prompts.chat import ( - ChatPromptTemplate, - HumanMessagePromptTemplate, - SystemMessagePromptTemplate, -) -from langchain.schema import HumanMessage, SystemMessage - -chat = ChatOpenAI( - openai_api_base="http://0.0.0.0:4000", - model = "gpt-3.5-turbo", - temperature=0.1, - extra_body={ - "metadata": { - "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] - } - } -) - -messages = [ - SystemMessage( - content="You are a helpful assistant that im using to make a test request to." - ), - HumanMessage( - content="test from litellm. tell me why it's amazing in 1 sentence" - ), -] -response = chat(messages) - -print(response) -``` - - - - - -### Viewing Spend per tag - -#### `/spend/tags` Request Format -```shell -curl -X GET "http://0.0.0.0:4000/spend/tags" \ --H "Authorization: Bearer sk-1234" -``` - -#### `/spend/tags`Response Format -```shell -[ - { - "individual_request_tag": "model-anthropic-claude-v2.1", - "log_count": 6, - "total_spend": 0.000672 - }, - { - "individual_request_tag": "app-ishaan-local", - "log_count": 4, - "total_spend": 0.000448 - }, - { - "individual_request_tag": "app-ishaan-prod", - "log_count": 2, - "total_spend": 0.000224 - } -] - -``` - - - - -## Swagger Docs - Custom Routes + Branding - -:::info - -Requires a LiteLLM Enterprise key to use. Request one [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) - -::: - -Set LiteLLM Key in your environment - -```bash -LITELLM_LICENSE="" -``` - -### Customize Title + Description - -In your environment, set: - -```bash -DOCS_TITLE="TotalGPT" -DOCS_DESCRIPTION="Sample Company Description" -``` - -### Customize Routes - -Hide admin routes from users. - -In your environment, set: - -```bash -DOCS_FILTERED="True" # only shows openai routes to user -``` - - \ No newline at end of file + \ No newline at end of file diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 538a81d4b..a55e42d55 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -3,22 +3,612 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🔎 Logging - Custom Callbacks, DataDog, Langfuse, s3 Bucket, Sentry, OpenTelemetry, Athina, Azure Content-Safety +# 🪢 Logging - Langfuse, OpenTelemetry, Custom Callbacks, DataDog, s3 Bucket, Sentry, Athina, Azure Content-Safety -Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTelemetry, LangFuse, DynamoDB, s3 Bucket +Log Proxy Input, Output, Exceptions using Langfuse, OpenTelemetry, Custom Callbacks, DataDog, DynamoDB, s3 Bucket +- [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) +- [Logging with OpenTelemetry (OpenTelemetry)](#logging-proxy-inputoutput-in-opentelemetry-format) - [Async Custom Callbacks](#custom-callback-class-async) - [Async Custom Callback APIs](#custom-callback-apis-async) -- [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) - [Logging to OpenMeter](#logging-proxy-inputoutput---langfuse) - [Logging to s3 Buckets](#logging-proxy-inputoutput---s3-buckets) - [Logging to DataDog](#logging-proxy-inputoutput---datadog) - [Logging to DynamoDB](#logging-proxy-inputoutput---dynamodb) - [Logging to Sentry](#logging-proxy-inputoutput---sentry) -- [Logging to Traceloop (OpenTelemetry)](#logging-proxy-inputoutput-traceloop-opentelemetry) - [Logging to Athina](#logging-proxy-inputoutput-athina) - [(BETA) Moderation with Azure Content-Safety](#moderation-with-azure-content-safety) +## Logging Proxy Input/Output - Langfuse +We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment + +**Step 1** Install langfuse + +```shell +pip install langfuse>=2.0.0 +``` + +**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["langfuse"] +``` + +**Step 3**: Set required env variables for logging to langfuse +```shell +export LANGFUSE_PUBLIC_KEY="pk_kk" +export LANGFUSE_SECRET_KEY="sk_ss" +# Optional, defaults to https://cloud.langfuse.com +export LANGFUSE_HOST="https://xxx.langfuse.com" +``` + +**Step 4**: Start the proxy, make a test request + +Start proxy +```shell +litellm --config config.yaml --debug +``` + +Test Request +``` +litellm --test +``` + +Expected output on Langfuse + + + +### Logging Metadata to Langfuse + + + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": { + "generation_name": "ishaan-test-generation", + "generation_id": "gen-id22", + "trace_id": "trace-id22", + "trace_user_id": "user-id2" + } +}' +``` + + + +Set `extra_body={"metadata": { }}` to `metadata` you want to pass + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "generation_name": "ishaan-generation-openai-client", + "generation_id": "openai-client-gen-id22", + "trace_id": "openai-client-trace-id22", + "trace_user_id": "openai-client-user-id2" + } + } +) + +print(response) +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "metadata": { + "generation_name": "ishaan-generation-langchain-client", + "generation_id": "langchain-client-gen-id22", + "trace_id": "langchain-client-trace-id22", + "trace_user_id": "langchain-client-user-id2" + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + +### Team based Logging to Langfuse + +**Example:** + +This config would send langfuse logs to 2 different langfuse projects, based on the team id + +```yaml +litellm_settings: + default_team_settings: + - team_id: my-secret-project + success_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_1 # Project 1 + langfuse_secret: os.environ/LANGFUSE_PRIVATE_KEY_1 # Project 1 + - team_id: ishaans-secret-project + success_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_2 # Project 2 + langfuse_secret: os.environ/LANGFUSE_SECRET_2 # Project 2 +``` + +Now, when you [generate keys](./virtual_keys.md) for this team-id + +```bash +curl -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{"team_id": "ishaans-secret-project"}' +``` + +All requests made with these keys will log data to their team-specific logging. + +### Redacting Messages, Response Content from Langfuse Logging + +Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["langfuse"] + turn_off_message_logging: True +``` + +### 🔧 Debugging - Viewing RAW CURL sent from LiteLLM to provider + +Use this when you want to view the RAW curl request sent from LiteLLM to the LLM API + + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": { + "log_raw_request": true + } +}' +``` + + + +Set `extra_body={"metadata": {"log_raw_request": True }}` to `metadata` you want to pass + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "log_raw_request": True + } + } +) + +print(response) +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "metadata": { + "log_raw_request": True + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + +**Expected Output on Langfuse** + +You will see `raw_request` in your Langfuse Metadata. This is the RAW CURL command sent from LiteLLM to your LLM API provider + + + + +## Logging Proxy Input/Output in OpenTelemetry format + +:::info + +[Optional] Customize OTEL Service Name and OTEL TRACER NAME by setting the following variables in your environment + +```shell +OTEL_TRACER_NAME= # default="litellm" +OTEL_SERVICE_NAME=` # default="litellm" +``` + +::: + + + + + + + +**Step 1:** Set callbacks and env vars + +Add the following to your env + +```shell +OTEL_EXPORTER="console" +``` + +Add `otel` as a callback on your `litellm_config.yaml` + +```shell +litellm_settings: + callbacks: ["otel"] +``` + + +**Step 2**: Start the proxy, make a test request + +Start proxy + +```shell +litellm --config config.yaml --detailed_debug +``` + +Test Request + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' +``` + +**Step 3**: **Expect to see the following logged on your server logs / console** + +This is the Span from OTEL Logging + +```json +{ + "name": "litellm-acompletion", + "context": { + "trace_id": "0x8d354e2346060032703637a0843b20a3", + "span_id": "0xd8d3476a2eb12724", + "trace_state": "[]" + }, + "kind": "SpanKind.INTERNAL", + "parent_id": null, + "start_time": "2024-06-04T19:46:56.415888Z", + "end_time": "2024-06-04T19:46:56.790278Z", + "status": { + "status_code": "OK" + }, + "attributes": { + "model": "llama3-8b-8192" + }, + "events": [], + "links": [], + "resource": { + "attributes": { + "service.name": "litellm" + }, + "schema_url": "" + } +} +``` + + + + + + +#### Quick Start - Log to Honeycomb + +**Step 1:** Set callbacks and env vars + +Add the following to your env + +```shell +OTEL_EXPORTER="otlp_http" +OTEL_ENDPOINT="https://api.honeycomb.io/v1/traces" +OTEL_HEADERS="x-honeycomb-team=" +``` + +Add `otel` as a callback on your `litellm_config.yaml` + +```shell +litellm_settings: + callbacks: ["otel"] +``` + + +**Step 2**: Start the proxy, make a test request + +Start proxy + +```shell +litellm --config config.yaml --detailed_debug +``` + +Test Request + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' +``` + + + + + + + +#### Quick Start - Log to OTEL Collector + +**Step 1:** Set callbacks and env vars + +Add the following to your env + +```shell +OTEL_EXPORTER="otlp_http" +OTEL_ENDPOINT="http:/0.0.0.0:4317" +OTEL_HEADERS="x-honeycomb-team=" # Optional +``` + +Add `otel` as a callback on your `litellm_config.yaml` + +```shell +litellm_settings: + callbacks: ["otel"] +``` + + +**Step 2**: Start the proxy, make a test request + +Start proxy + +```shell +litellm --config config.yaml --detailed_debug +``` + +Test Request + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' +``` + + + + + + +#### Quick Start - Log to OTEL GRPC Collector + +**Step 1:** Set callbacks and env vars + +Add the following to your env + +```shell +OTEL_EXPORTER="otlp_grpc" +OTEL_ENDPOINT="http:/0.0.0.0:4317" +OTEL_HEADERS="x-honeycomb-team=" # Optional +``` + +Add `otel` as a callback on your `litellm_config.yaml` + +```shell +litellm_settings: + callbacks: ["otel"] +``` + + +**Step 2**: Start the proxy, make a test request + +Start proxy + +```shell +litellm --config config.yaml --detailed_debug +``` + +Test Request + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' +``` + + + + + + +#### Quick Start - Log to Traceloop + +**Step 1:** Install the `traceloop-sdk` SDK + +```shell +pip install traceloop-sdk==0.21.2 +``` + +**Step 2:** Add `traceloop` as a success_callback + +```shell +litellm_settings: + success_callback: ["traceloop"] + +environment_variables: + TRACELOOP_API_KEY: "XXXXX" +``` + + +**Step 3**: Start the proxy, make a test request + +Start proxy + +```shell +litellm --config config.yaml --detailed_debug +``` + +Test Request + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' +``` + + + + + +** 🎉 Expect to see this trace logged in your OTEL collector** + + + + ## Custom Callback Class [Async] Use this when you want to run custom callbacks in `python` @@ -402,197 +992,6 @@ litellm_settings: Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API -## Logging Proxy Input/Output - Langfuse -We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment - -**Step 1** Install langfuse - -```shell -pip install langfuse>=2.0.0 -``` - -**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo -litellm_settings: - success_callback: ["langfuse"] -``` - -**Step 3**: Set required env variables for logging to langfuse -```shell -export LANGFUSE_PUBLIC_KEY="pk_kk" -export LANGFUSE_SECRET_KEY="sk_ss -``` - -**Step 4**: Start the proxy, make a test request - -Start proxy -```shell -litellm --config config.yaml --debug -``` - -Test Request -``` -litellm --test -``` - -Expected output on Langfuse - - - -### Logging Metadata to Langfuse - - - - - - -Pass `metadata` as part of the request body - -```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - "metadata": { - "generation_name": "ishaan-test-generation", - "generation_id": "gen-id22", - "trace_id": "trace-id22", - "trace_user_id": "user-id2" - } -}' -``` - - - -Set `extra_body={"metadata": { }}` to `metadata` you want to pass - -```python -import openai -client = openai.OpenAI( - api_key="anything", - base_url="http://0.0.0.0:4000" -) - -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } - ], - extra_body={ - "metadata": { - "generation_name": "ishaan-generation-openai-client", - "generation_id": "openai-client-gen-id22", - "trace_id": "openai-client-trace-id22", - "trace_user_id": "openai-client-user-id2" - } - } -) - -print(response) -``` - - - -```python -from langchain.chat_models import ChatOpenAI -from langchain.prompts.chat import ( - ChatPromptTemplate, - HumanMessagePromptTemplate, - SystemMessagePromptTemplate, -) -from langchain.schema import HumanMessage, SystemMessage - -chat = ChatOpenAI( - openai_api_base="http://0.0.0.0:4000", - model = "gpt-3.5-turbo", - temperature=0.1, - extra_body={ - "metadata": { - "generation_name": "ishaan-generation-langchain-client", - "generation_id": "langchain-client-gen-id22", - "trace_id": "langchain-client-trace-id22", - "trace_user_id": "langchain-client-user-id2" - } - } -) - -messages = [ - SystemMessage( - content="You are a helpful assistant that im using to make a test request to." - ), - HumanMessage( - content="test from litellm. tell me why it's amazing in 1 sentence" - ), -] -response = chat(messages) - -print(response) -``` - - - - - -### Team based Logging to Langfuse - -**Example:** - -This config would send langfuse logs to 2 different langfuse projects, based on the team id - -```yaml -litellm_settings: - default_team_settings: - - team_id: my-secret-project - success_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_1 # Project 1 - langfuse_secret: os.environ/LANGFUSE_PRIVATE_KEY_1 # Project 1 - - team_id: ishaans-secret-project - success_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_2 # Project 2 - langfuse_secret: os.environ/LANGFUSE_SECRET_2 # Project 2 -``` - -Now, when you [generate keys](./virtual_keys.md) for this team-id - -```bash -curl -X POST 'http://0.0.0.0:4000/key/generate' \ --H 'Authorization: Bearer sk-1234' \ --H 'Content-Type: application/json' \ --d '{"team_id": "ishaans-secret-project"}' -``` - -All requests made with these keys will log data to their team-specific logging. - -### Redacting Messages, Response Content from Langfuse Logging - -Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. - -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo -litellm_settings: - success_callback: ["langfuse"] - turn_off_message_logging: True -``` - - - ## Logging Proxy Cost + Usage - OpenMeter Bill customers according to their LLM API usage with [OpenMeter](../observability/openmeter.md) @@ -915,86 +1314,6 @@ Test Request litellm --test ``` -## Logging Proxy Input/Output in OpenTelemetry format using Traceloop's OpenLLMetry - -[OpenLLMetry](https://github.com/traceloop/openllmetry) _(built and maintained by Traceloop)_ is a set of extensions -built on top of [OpenTelemetry](https://opentelemetry.io/) that gives you complete observability over your LLM -application. Because it uses OpenTelemetry under the -hood, [it can be connected to various observability solutions](https://www.traceloop.com/docs/openllmetry/integrations/introduction) -like: - -* [Traceloop](https://www.traceloop.com/docs/openllmetry/integrations/traceloop) -* [Axiom](https://www.traceloop.com/docs/openllmetry/integrations/axiom) -* [Azure Application Insights](https://www.traceloop.com/docs/openllmetry/integrations/azure) -* [Datadog](https://www.traceloop.com/docs/openllmetry/integrations/datadog) -* [Dynatrace](https://www.traceloop.com/docs/openllmetry/integrations/dynatrace) -* [Grafana Tempo](https://www.traceloop.com/docs/openllmetry/integrations/grafana) -* [Honeycomb](https://www.traceloop.com/docs/openllmetry/integrations/honeycomb) -* [HyperDX](https://www.traceloop.com/docs/openllmetry/integrations/hyperdx) -* [Instana](https://www.traceloop.com/docs/openllmetry/integrations/instana) -* [New Relic](https://www.traceloop.com/docs/openllmetry/integrations/newrelic) -* [OpenTelemetry Collector](https://www.traceloop.com/docs/openllmetry/integrations/otel-collector) -* [Service Now Cloud Observability](https://www.traceloop.com/docs/openllmetry/integrations/service-now) -* [Sentry](https://www.traceloop.com/docs/openllmetry/integrations/sentry) -* [SigNoz](https://www.traceloop.com/docs/openllmetry/integrations/signoz) -* [Splunk](https://www.traceloop.com/docs/openllmetry/integrations/splunk) - -We will use the `--config` to set `litellm.success_callback = ["traceloop"]` to achieve this, steps are listed below. - -**Step 1:** Install the SDK - -```shell -pip install traceloop-sdk -``` - -**Step 2:** Configure Environment Variable for trace exporting - -You will need to configure where to export your traces. Environment variables will control this, example: For Traceloop -you should use `TRACELOOP_API_KEY`, whereas for Datadog you use `TRACELOOP_BASE_URL`. For more -visit [the Integrations Catalog](https://www.traceloop.com/docs/openllmetry/integrations/introduction). - -If you are using Datadog as the observability solutions then you can set `TRACELOOP_BASE_URL` as: - -```shell -TRACELOOP_BASE_URL=http://:4318 -``` - -**Step 3**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` - -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - api_key: my-fake-key # replace api_key with actual key -litellm_settings: - success_callback: [ "traceloop" ] -``` - -**Step 4**: Start the proxy, make a test request - -Start proxy - -```shell -litellm --config config.yaml --debug -``` - -Test Request - -``` -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data ' { - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ] - }' -``` - ## Logging Proxy Input/Output Athina [Athina](https://athina.ai/) allows you to log LLM Input/Output for monitoring, analytics, and observability. diff --git a/docs/my-website/docs/proxy/multiple_admins.md b/docs/my-website/docs/proxy/multiple_admins.md new file mode 100644 index 000000000..376ff0174 --- /dev/null +++ b/docs/my-website/docs/proxy/multiple_admins.md @@ -0,0 +1,99 @@ +# ✨ Attribute Management changes to Users + +Call management endpoints on behalf of a user. (Useful when connecting proxy to your development platform). + + +:::tip + +Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + +## 1. Switch on audit Logs +Add `store_audit_logs` to your litellm config.yaml and then start the proxy +```shell +litellm_settings: + store_audit_logs: true +``` + +## 2. Set `LiteLLM-Changed-By` in request headers + +Set the 'user_id' in request headers, when calling a management endpoint. [View Full List](https://litellm-api.up.railway.app/#/team%20management). + +- Update Team budget with master key. +- Attribute change to 'krrish@berri.ai'. + +**👉 Key change:** Passing `-H 'LiteLLM-Changed-By: krrish@berri.ai'` + +```shell +curl -X POST 'http://0.0.0.0:4000/team/update' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'LiteLLM-Changed-By: krrish@berri.ai' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_id" : "8bf18b11-7f52-4717-8e1f-7c65f9d01e52", + "max_budget": 2000 + }' +``` + +## 3. Emitted Audit Log + +```bash +{ + "id": "bd136c28-edd0-4cb6-b963-f35464cf6f5a", + "updated_at": "2024-06-08 23:41:14.793", + "changed_by": "krrish@berri.ai", # 👈 CHANGED BY + "changed_by_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b", + "action": "updated", + "table_name": "LiteLLM_TeamTable", + "object_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52", + "before_value": { + "spend": 0, + "max_budget": 0, + }, + "updated_values": { + "team_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52", + "max_budget": 2000 # 👈 CHANGED TO + }, + } +``` + +## API SPEC of Audit Log + + +### `id` +- **Type:** `String` +- **Description:** This is the unique identifier for each audit log entry. It is automatically generated as a UUID (Universally Unique Identifier) by default. + +### `updated_at` +- **Type:** `DateTime` +- **Description:** This field stores the timestamp of when the audit log entry was created or updated. It is automatically set to the current date and time by default. + +### `changed_by` +- **Type:** `String` +- **Description:** The `user_id` that performed the audited action. If `LiteLLM-Changed-By` Header is passed then `changed_by=` + +### `changed_by_api_key` +- **Type:** `String` +- **Description:** This field stores the hashed API key that was used to perform the audited action. If left blank, it defaults to an empty string. + +### `action` +- **Type:** `String` +- **Description:** The type of action that was performed. One of "create", "update", or "delete". + +### `table_name` +- **Type:** `String` +- **Description:** This field stores the name of the table that was affected by the audited action. It can be one of the following values: `LiteLLM_TeamTable`, `LiteLLM_UserTable`, `LiteLLM_VerificationToken` + + +### `object_id` +- **Type:** `String` +- **Description:** This field stores the ID of the object that was affected by the audited action. It can be the key ID, team ID, user ID + +### `before_value` +- **Type:** `Json?` +- **Description:** This field stores the value of the row before the audited action was performed. It is optional and can be null. + +### `updated_values` +- **Type:** `Json?` +- **Description:** This field stores the values of the row that were updated after the audited action was performed \ No newline at end of file diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 35c8c575b..587164fe6 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -21,6 +21,7 @@ general_settings: litellm_settings: set_verbose: False # Switch off Debug Logging, ensure your logs do not have any debugging on + json_logs: true # Get debug logs in json format ``` Set slack webhook url in your env @@ -28,6 +29,11 @@ Set slack webhook url in your env export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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" ``` +Turn off FASTAPI's default info logs +```bash +export LITELLM_LOG="ERROR" +``` + :::info Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) diff --git a/docs/my-website/docs/proxy/quick_start.md b/docs/my-website/docs/proxy/quick_start.md index 050d9b598..4ee4d8831 100644 --- a/docs/my-website/docs/proxy/quick_start.md +++ b/docs/my-website/docs/proxy/quick_start.md @@ -24,6 +24,15 @@ $ litellm --model huggingface/bigcode/starcoder #INFO: Proxy running on http://0.0.0.0:4000 ``` + +:::info + +Run with `--detailed_debug` if you need detailed debug logs + +```shell +$ litellm --model huggingface/bigcode/starcoder --detailed_debug +::: + ### Test In a new shell, run, this will make an `openai.chat.completions` request. Ensure you're using openai v1.0.0+ ```shell diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index e39a6765f..d4ab1a2c7 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -2,18 +2,13 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🔥 Fallbacks, Retries, Timeouts, Load Balancing +# 🔥 Load Balancing, Fallbacks, Retries, Timeouts -Retry call with multiple instances of the same model. - -If a call fails after num_retries, fall back to another model group. - -If the error is a context window exceeded error, fall back to a larger model group (if given). - -[**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/router.py) +- Quick Start [load balancing](#test---load-balancing) +- Quick Start [client side fallbacks](#test---client-side-fallbacks) ## Quick Start - Load Balancing -### Step 1 - Set deployments on config +#### Step 1 - Set deployments on config **Example config below**. Here requests with `model=gpt-3.5-turbo` will be routed across multiple instances of `azure/gpt-3.5-turbo` ```yaml @@ -38,50 +33,214 @@ model_list: rpm: 1440 ``` -### Step 2: Start Proxy with config +#### Step 2: Start Proxy with config ```shell $ litellm --config /path/to/config.yaml ``` -### Step 3: Use proxy - Call a model group [Load Balancing] -Curl Command +### Test - Load Balancing + +Here requests with model=gpt-3.5-turbo will be routed across multiple instances of azure/gpt-3.5-turbo + +👉 Key Change: `model="gpt-3.5-turbo"` + +**Check the `model_id` in Response Headers to make sure the requests are being load balanced** + + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ] +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body + ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "gpt-3.5-turbo", - "messages": [ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ { - "role": "user", - "content": "what llm are you" + "role": "user", + "content": "what llm are you" } - ], - } -' + ] +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="gpt-3.5-turbo", +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) ``` -### Usage - Call a specific model deployment -If you want to call a specific model defined in the `config.yaml`, you can call the `litellm_params: model` + + + + + +### Test - Client Side Fallbacks +In this request the following will occur: +1. The request to `model="zephyr-beta"` will fail +2. litellm proxy will loop through all the model_groups specified in `fallbacks=["gpt-3.5-turbo"]` +3. The request to `model="gpt-3.5-turbo"` will succeed and the client making the request will get a response from gpt-3.5-turbo + +👉 Key Change: `"fallbacks": ["gpt-3.5-turbo"]` + + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="zephyr-beta", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "fallbacks": ["gpt-3.5-turbo"] + } +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "zephyr-beta"", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "fallbacks": ["gpt-3.5-turbo"] +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="zephyr-beta", + extra_body={ + "fallbacks": ["gpt-3.5-turbo"] + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + + + + -## Fallbacks + Retries + Timeouts + Cooldowns +## Advanced +### Fallbacks + Retries + Timeouts + Cooldowns **Set via config** ```yaml @@ -114,44 +273,7 @@ litellm_settings: context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. ``` - -**Set dynamically** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "zephyr-beta", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "num_retries": 2, - "timeout": 10 - } -' -``` - -### Test it! - - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data-raw '{ - "model": "zephyr-beta", # 👈 MODEL NAME to fallback from - "messages": [ - {"role": "user", "content": "what color is red"} - ], - "mock_testing_fallbacks": true - }' -``` - -## Advanced - Context Window Fallbacks (Pre-Call Checks + Fallbacks) +### Context Window Fallbacks (Pre-Call Checks + Fallbacks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -287,7 +409,7 @@ print(response) -## Advanced - EU-Region Filtering (Pre-Call Checks) +### EU-Region Filtering (Pre-Call Checks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -350,7 +472,7 @@ print(response) print(f"response.headers.get('x-litellm-model-api-base')") ``` -## Advanced - Custom Timeouts, Stream Timeouts - Per Model +### Custom Timeouts, Stream Timeouts - Per Model For each model you can set `timeout` & `stream_timeout` under `litellm_params` ```yaml model_list: @@ -379,7 +501,7 @@ $ litellm --config /path/to/config.yaml ``` -## Advanced - Setting Dynamic Timeouts - Per Request +### Setting Dynamic Timeouts - Per Request LiteLLM Proxy supports setting a `timeout` per request diff --git a/docs/my-website/docs/proxy/self_serve.md b/docs/my-website/docs/proxy/self_serve.md new file mode 100644 index 000000000..568e6541a --- /dev/null +++ b/docs/my-website/docs/proxy/self_serve.md @@ -0,0 +1,126 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 🤗 UI - Self-Serve + +Allow users to creat their own keys on [Proxy UI](./ui.md). + +1. Add user with permissions to a team on proxy + + + + +Go to `Internal Users` -> `+New User` + + + + + + +Create a new Internal User on LiteLLM and assign them the role `internal_user`. + +```bash +curl -X POST '/user/new' \ +-H 'Authorization: Bearer ' \ +-H 'Content-Type: application/json' \ +-D '{ + "user_email": "krrishdholakia@gmail.com", + "user_role": "internal_user" # 👈 THIS ALLOWS USER TO CREATE/VIEW/DELETE THEIR OWN KEYS + SEE THEIR SPEND +}' +``` + +Expected Response + +```bash +{ + "user_id": "e9d45c7c-b20b-4ff8-ae76-3f479a7b1d7d", 👈 USE IN STEP 2 + "user_email": "", + "user_role": "internal_user", + ... +} +``` + +Here's the available UI roles for a LiteLLM Internal User: + +Admin Roles: + - `proxy_admin`: admin over the platform + - `proxy_admin_viewer`: can login, view all keys, view all spend. **Cannot** create/delete keys, add new users. + +Internal User Roles: + - `internal_user`: can login, view/create/delete their own keys, view their spend. **Cannot** add new users. + - `internal_user_viewer`: can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users. + + + + +2. Share invitation link with user + + + + +Copy the invitation link with the user + + + + + + +```bash +curl -X POST '/invitation/new' \ +-H 'Authorization: Bearer ' \ +-H 'Content-Type: application/json' \ +-D '{ + "user_id": "e9d45c7c-b20b..." # 👈 USER ID FROM STEP 1 +}' +``` + +Expected Response + +```bash +{ + "id": "a2f0918f-43b0-4770-a664-96ddd192966e", + "user_id": "e9d45c7c-b20b..", + "is_accepted": false, + "accepted_at": null, + "expires_at": "2024-06-13T00:02:16.454000Z", # 👈 VALID FOR 7d + "created_at": "2024-06-06T00:02:16.454000Z", + "created_by": "116544810872468347480", + "updated_at": "2024-06-06T00:02:16.454000Z", + "updated_by": "116544810872468347480" +} +``` + +Invitation Link: + +```bash +http://0.0.0.0:4000/ui/onboarding?id=a2f0918f-43b0-4770-a664-96ddd192966e + +# /ui/onboarding?id= +``` + + + + +:::info + +Use [Email Notifications](./email.md) to email users onboarding links + +::: + +3. User logs in via email + password auth + + + + + +:::info + +LiteLLM Enterprise: Enable [SSO login](./ui.md#setup-ssoauth-for-ui) + +::: + +4. User can now create their own keys + + + \ No newline at end of file diff --git a/docs/my-website/docs/proxy/ui.md b/docs/my-website/docs/proxy/ui.md index 19ffd4ca2..6a828075a 100644 --- a/docs/my-website/docs/proxy/ui.md +++ b/docs/my-website/docs/proxy/ui.md @@ -2,10 +2,9 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# [BETA] Proxy UI -### **Create + delete keys through a UI** +# [BETA] UI - Admin -[Let users create their own keys](#setup-ssoauth-for-ui) +Create keys, track spend, add models without worrying about the config / CRUD endpoints. :::info diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 556ae7f92..77fbcb444 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -13,7 +13,7 @@ Requirements: You can set budgets at 3 levels: - For the proxy - For an internal user -- For an end-user +- For a customer (end-user) - For a key - For a key (model specific budgets) @@ -63,7 +63,7 @@ You can: - Add budgets to Teams -#### **Add budgets to users** +#### **Add budgets to teams** ```shell curl --location 'http://localhost:4000/team/new' \ --header 'Authorization: Bearer ' \ @@ -102,6 +102,22 @@ curl --location 'http://localhost:4000/team/new' \ "budget_reset_at": null } ``` + +#### **Add budget duration to teams** + +`budget_duration`: Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). + +``` +curl 'http://0.0.0.0:4000/team/new' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "team_alias": "my-new-team_4", + "members_with_roles": [{"role": "admin", "user_id": "5c4a0aa3-a1e1-43dc-bd87-3c2da8382a3a"}], + "budget_duration": 10s, +}' +``` + @@ -173,7 +189,7 @@ curl --location 'http://localhost:4000/chat/completions' \ ``` - + Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user** @@ -223,7 +239,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ Error ```shell -{"error":{"message":"Authentication Error, ExceededBudget: User ishaan3 has exceeded their budget. Current spend: 0.0008869999999999999; Max Budget: 0.0001","type":"auth_error","param":"None","code":401}}% +{"error":{"message":"Budget has been exceeded: User ishaan3 has exceeded their budget. Current spend: 0.0008869999999999999; Max Budget: 0.0001","type":"auth_error","param":"None","code":401}}% ``` @@ -452,7 +468,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ ``` - + :::info @@ -477,12 +493,12 @@ curl --location 'http://0.0.0.0:4000/budget/new' \ ``` -#### Step 2. Create `End-User` with Budget +#### Step 2. Create `Customer` with Budget -We use `budget_id="free-tier"` from Step 1 when creating this new end user +We use `budget_id="free-tier"` from Step 1 when creating this new customers ```shell -curl --location 'http://0.0.0.0:4000/end_user/new' \ +curl --location 'http://0.0.0.0:4000/customer/new' \ --header 'Authorization: Bearer sk-1234' \ --header 'Content-Type: application/json' \ --data '{ @@ -492,7 +508,7 @@ curl --location 'http://0.0.0.0:4000/end_user/new' \ ``` -#### Step 3. Pass end user id in `/chat/completions` requests +#### Step 3. Pass `user_id` id in `/chat/completions` requests Pass the `user_id` from Step 2 as `user="palantir"` diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 5ba3221c9..d91912644 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -713,26 +713,43 @@ response = router.completion(model="gpt-3.5-turbo", messages=messages) print(f"response: {response}") ``` -#### Retries based on Error Type +### [Advanced]: Custom Retries, Cooldowns based on Error Type -Use `RetryPolicy` if you want to set a `num_retries` based on the Exception receieved +- Use `RetryPolicy` if you want to set a `num_retries` based on the Exception receieved +- Use `AllowedFailsPolicy` to set a custom number of `allowed_fails`/minute before cooling down a deployment Example: -- 4 retries for `ContentPolicyViolationError` -- 0 retries for `RateLimitErrors` + +```python +retry_policy = RetryPolicy( + ContentPolicyViolationErrorRetries=3, # run 3 retries for ContentPolicyViolationErrors + AuthenticationErrorRetries=0, # run 0 retries for AuthenticationErrorRetries +) + +allowed_fails_policy = AllowedFailsPolicy( + ContentPolicyViolationErrorAllowedFails=1000, # Allow 1000 ContentPolicyViolationError before cooling down a deployment + RateLimitErrorAllowedFails=100, # Allow 100 RateLimitErrors before cooling down a deployment +) +``` Example Usage ```python -from litellm.router import RetryPolicy +from litellm.router import RetryPolicy, AllowedFailsPolicy + retry_policy = RetryPolicy( - ContentPolicyViolationErrorRetries=3, # run 3 retries for ContentPolicyViolationErrors - AuthenticationErrorRetries=0, # run 0 retries for AuthenticationErrorRetries + ContentPolicyViolationErrorRetries=3, # run 3 retries for ContentPolicyViolationErrors + AuthenticationErrorRetries=0, # run 0 retries for AuthenticationErrorRetries BadRequestErrorRetries=1, TimeoutErrorRetries=2, RateLimitErrorRetries=3, ) +allowed_fails_policy = AllowedFailsPolicy( + ContentPolicyViolationErrorAllowedFails=1000, # Allow 1000 ContentPolicyViolationError before cooling down a deployment + RateLimitErrorAllowedFails=100, # Allow 100 RateLimitErrors before cooling down a deployment +) + router = litellm.Router( model_list=[ { @@ -755,6 +772,7 @@ router = litellm.Router( }, ], retry_policy=retry_policy, + allowed_fails_policy=allowed_fails_policy, ) response = await router.acompletion( diff --git a/docs/my-website/docs/scheduler.md b/docs/my-website/docs/scheduler.md new file mode 100644 index 000000000..e7943c459 --- /dev/null +++ b/docs/my-website/docs/scheduler.md @@ -0,0 +1,175 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# [BETA] Request Prioritization + +:::info + +Beta feature. Use for testing only. + +[Help us improve this](https://github.com/BerriAI/litellm/issues) +::: + +Prioritize LLM API requests in high-traffic. + +- Add request to priority queue +- Poll queue, to check if request can be made. Returns 'True': + * if there's healthy deployments + * OR if request is at top of queue +- Priority - The lower the number, the higher the priority: + * e.g. `priority=0` > `priority=2000` + +## Quick Start + +```python +from litellm import Router + +router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "mock_response": "Hello world this is Macintosh!", # fakes the LLM API call + "rpm": 1, + }, + }, + ], + timeout=2, # timeout request if takes > 2s + routing_strategy="usage-based-routing-v2", + polling_interval=0.03 # poll queue every 3ms if no healthy deployments +) + +try: + _response = await router.schedule_acompletion( # 👈 ADDS TO QUEUE + POLLS + MAKES CALL + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey!"}], + priority=0, # 👈 LOWER IS BETTER + ) +except Exception as e: + print("didn't make request") +``` + +## LiteLLM Proxy + +To prioritize requests on LiteLLM Proxy call our beta openai-compatible `http://localhost:4000/queue` endpoint. + + + + +```curl +curl -X POST 'http://localhost:4000/queue/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "gpt-3.5-turbo-fake-model", + "messages": [ + { + "role": "user", + "content": "what is the meaning of the universe? 1234" + }], + "priority": 0 👈 SET VALUE HERE +}' +``` + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "priority": 0 👈 SET VALUE HERE + } +) + +print(response) +``` + + + + +## Advanced - Redis Caching + +Use redis caching to do request prioritization across multiple instances of LiteLLM. + +### SDK +```python +from litellm import Router + +router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "mock_response": "Hello world this is Macintosh!", # fakes the LLM API call + "rpm": 1, + }, + }, + ], + ### REDIS PARAMS ### + redis_host=os.environ["REDIS_HOST"], + redis_password=os.environ["REDIS_PASSWORD"], + redis_port=os.environ["REDIS_PORT"], +) + +try: + _response = await router.schedule_acompletion( # 👈 ADDS TO QUEUE + POLLS + MAKES CALL + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey!"}], + priority=0, # 👈 LOWER IS BETTER + ) +except Exception as e: + print("didn't make request") +``` + +### PROXY + +```yaml +model_list: + - model_name: gpt-3.5-turbo-fake-model + litellm_params: + model: gpt-3.5-turbo + mock_response: "hello world!" + api_key: my-good-key + +router_settings: + redis_host; os.environ/REDIS_HOST + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT +``` + +```bash +$ litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000s +``` + +```bash +curl -X POST 'http://localhost:4000/queue/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "gpt-3.5-turbo-fake-model", + "messages": [ + { + "role": "user", + "content": "what is the meaning of the universe? 1234" + }], + "priority": 0 👈 SET VALUE HERE +}' +``` \ No newline at end of file diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index 2b945837a..08c2e89d1 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -1,11 +1,31 @@ # Secret Manager LiteLLM supports reading secrets from Azure Key Vault and Infisical +- AWS Key Managemenet Service +- AWS Secret Manager - [Azure Key Vault](#azure-key-vault) - Google Key Management Service - [Infisical Secret Manager](#infisical-secret-manager) - [.env Files](#env-files) +## AWS Key Management Service + +Use AWS KMS to storing a hashed copy of your Proxy Master Key in the environment. + +```bash +export LITELLM_MASTER_KEY="djZ9xjVaZ..." # 👈 ENCRYPTED KEY +export AWS_REGION_NAME="us-west-2" +``` + +```yaml +general_settings: + key_management_system: "aws_kms" + key_management_settings: + hosted_keys: ["LITELLM_MASTER_KEY"] # 👈 WHICH KEYS ARE STORED ON KMS +``` + +[**See Decryption Code**](https://github.com/BerriAI/litellm/blob/a2da2a8f168d45648b61279d4795d647d94f90c9/litellm/utils.py#L10182) + ## AWS Secret Manager Store your proxy keys in AWS Secret Manager. diff --git a/docs/my-website/docs/text_to_speech.md b/docs/my-website/docs/text_to_speech.md new file mode 100644 index 000000000..f4adf15eb --- /dev/null +++ b/docs/my-website/docs/text_to_speech.md @@ -0,0 +1,87 @@ +# Text to Speech + +## Quick Start + +```python +from pathlib import Path +from litellm import speech +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = speech( + model="openai/tts-1", + voice="alloy", + input="the quick brown fox jumped over the lazy dogs", + api_base=None, + api_key=None, + organization=None, + project=None, + max_retries=1, + timeout=600, + client=None, + optional_params={}, + ) +response.stream_to_file(speech_file_path) +``` + +## Async Usage + +```python +from litellm import aspeech +from pathlib import Path +import os, asyncio + +os.environ["OPENAI_API_KEY"] = "sk-.." + +async def test_async_speech(): + speech_file_path = Path(__file__).parent / "speech.mp3" + response = await litellm.aspeech( + model="openai/tts-1", + voice="alloy", + input="the quick brown fox jumped over the lazy dogs", + api_base=None, + api_key=None, + organization=None, + project=None, + max_retries=1, + timeout=600, + client=None, + optional_params={}, + ) + response.stream_to_file(speech_file_path) + +asyncio.run(test_async_speech()) +``` + +## Proxy Usage + +LiteLLM provides an openai-compatible `/audio/speech` endpoint for Text-to-speech calls. + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts-1", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy" + }' \ + --output speech.mp3 +``` + +**Setup** + +```bash +- model_name: tts + litellm_params: + model: openai/tts-1 + api_key: os.environ/OPENAI_API_KEY +``` + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` \ No newline at end of file diff --git a/docs/my-website/docs/troubleshoot.md b/docs/my-website/docs/troubleshoot.md index 75a610e0c..3ca57a570 100644 --- a/docs/my-website/docs/troubleshoot.md +++ b/docs/my-website/docs/troubleshoot.md @@ -9,12 +9,3 @@ Our emails ✉️ ishaan@berri.ai / krrish@berri.ai [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) -## Stable Version - -If you're running into problems with installation / Usage -Use the stable version of litellm - -```shell -pip install litellm==0.1.819 -``` - diff --git a/docs/my-website/docs/tutorials/finetuned_chat_gpt.md b/docs/my-website/docs/tutorials/finetuned_chat_gpt.md index 641c45b5f..5dde3b3ff 100644 --- a/docs/my-website/docs/tutorials/finetuned_chat_gpt.md +++ b/docs/my-website/docs/tutorials/finetuned_chat_gpt.md @@ -1,8 +1,8 @@ # Using Fine-Tuned gpt-3.5-turbo LiteLLM allows you to call `completion` with your fine-tuned gpt-3.5-turbo models -If you're trying to create your custom finetuned gpt-3.5-turbo model following along on this tutorial: https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset +If you're trying to create your custom fine-tuned gpt-3.5-turbo model following along on this tutorial: https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset -Once you've created your fine tuned model, you can call it with `litellm.completion()` +Once you've created your fine-tuned model, you can call it with `litellm.completion()` ## Usage ```python diff --git a/docs/my-website/img/add_internal_user.png b/docs/my-website/img/add_internal_user.png new file mode 100644 index 000000000..eb6c68b2b Binary files /dev/null and b/docs/my-website/img/add_internal_user.png differ diff --git a/docs/my-website/img/admin_ui_spend.png b/docs/my-website/img/admin_ui_spend.png new file mode 100644 index 000000000..6a7196f83 Binary files /dev/null and b/docs/my-website/img/admin_ui_spend.png differ diff --git a/docs/my-website/img/create_budget_modal.png b/docs/my-website/img/create_budget_modal.png new file mode 100644 index 000000000..0e307be5e Binary files /dev/null and b/docs/my-website/img/create_budget_modal.png differ diff --git a/docs/my-website/img/debug_langfuse.png b/docs/my-website/img/debug_langfuse.png new file mode 100644 index 000000000..8768fcd09 Binary files /dev/null and b/docs/my-website/img/debug_langfuse.png differ diff --git a/docs/my-website/img/invitation_link.png b/docs/my-website/img/invitation_link.png new file mode 100644 index 000000000..e65767327 Binary files /dev/null and b/docs/my-website/img/invitation_link.png differ diff --git a/docs/my-website/img/model_hub.png b/docs/my-website/img/model_hub.png new file mode 100644 index 000000000..1aafc993a Binary files /dev/null and b/docs/my-website/img/model_hub.png differ diff --git a/docs/my-website/img/ui_clean_login.png b/docs/my-website/img/ui_clean_login.png new file mode 100644 index 000000000..62c65d4ae Binary files /dev/null and b/docs/my-website/img/ui_clean_login.png differ diff --git a/docs/my-website/img/ui_self_serve_create_key.png b/docs/my-website/img/ui_self_serve_create_key.png new file mode 100644 index 000000000..4b83e9abf Binary files /dev/null and b/docs/my-website/img/ui_self_serve_create_key.png differ diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index 0738c4d09..48b541986 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -5975,9 +5975,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001519", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001519.tgz", - "integrity": "sha512-0QHgqR+Jv4bxHMp8kZ1Kn8CH55OikjKJ6JmKkZYP1F3D7w+lnFXF70nG5eNfsZS89jadi5Ywy5UCSKLAglIRkg==", + "version": "1.0.30001629", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001629.tgz", + "integrity": "sha512-c3dl911slnQhmxUIT4HhYzT7wnBK/XYpGnYLOj4nJBaRiw52Ibe7YxlDaAeRECvA786zCuExhxIUJ2K7nHMrBw==", "funding": [ { "type": "opencollective", diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 330a3c550..ff110bb62 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -36,23 +36,26 @@ const sidebars = { label: "📖 All Endpoints (Swagger)", href: "https://litellm-api.up.railway.app/", }, + "proxy/enterprise", "proxy/demo", "proxy/configs", "proxy/reliability", "proxy/cost_tracking", + "proxy/self_serve", "proxy/users", + "proxy/customers", "proxy/billing", "proxy/user_keys", - "proxy/enterprise", "proxy/virtual_keys", "proxy/alerting", { type: "category", - label: "Logging", + label: "🪢 Logging", items: ["proxy/logging", "proxy/streaming_logging"], }, "proxy/ui", "proxy/email", + "proxy/multiple_admins", "proxy/team_based_routing", "proxy/customer_routing", "proxy/token_auth", @@ -99,13 +102,16 @@ const sidebars = { }, { type: "category", - label: "Embedding(), Moderation(), Image Generation(), Audio Transcriptions()", + label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches()", items: [ "embedding/supported_embedding", "embedding/async_embedding", "embedding/moderation", "image_generation", - "audio_transcription" + "audio_transcription", + "text_to_speech", + "assistants", + "batches", ], }, { @@ -163,6 +169,7 @@ const sidebars = { }, "proxy/custom_pricing", "routing", + "scheduler", "rules", "set_keys", "budget_manager", @@ -248,6 +255,7 @@ const sidebars = { "projects/GPT Migrate", "projects/YiVal", "projects/LiteLLM Proxy", + "projects/llm_cord", ], }, ], diff --git a/docs/my-website/yarn.lock b/docs/my-website/yarn.lock index 3acc48153..e96d90e26 100644 --- a/docs/my-website/yarn.lock +++ b/docs/my-website/yarn.lock @@ -84,7 +84,7 @@ "@algolia/requester-common" "4.19.1" "@algolia/transporter" "4.19.1" -"@algolia/client-search@4.19.1": +"@algolia/client-search@>= 4.9.1 < 6", "@algolia/client-search@4.19.1": version "4.19.1" resolved "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.19.1.tgz" integrity sha512-mBecfMFS4N+yK/p0ZbK53vrZbL6OtWMk8YmnOv1i0LXx4pelY8TFhqKoTit3NPVPwoSNN0vdSN9dTu1xr1XOVw== @@ -146,13 +146,6 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" -"@babel/code-frame@7.10.4", "@babel/code-frame@^7.5.5": - version "7.10.4" - resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz" - integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg== - dependencies: - "@babel/highlight" "^7.10.4" - "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.8.3": version "7.22.13" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz" @@ -161,11 +154,39 @@ "@babel/highlight" "^7.22.13" chalk "^2.4.2" +"@babel/code-frame@^7.5.5", "@babel/code-frame@7.10.4": + version "7.10.4" + resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz" + integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg== + dependencies: + "@babel/highlight" "^7.10.4" + "@babel/compat-data@^7.22.5", "@babel/compat-data@^7.22.6", "@babel/compat-data@^7.22.9": version "7.22.9" resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.22.9.tgz" integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== +"@babel/core@^7.0.0", "@babel/core@^7.0.0-0", "@babel/core@^7.0.0-0 || ^8.0.0-0 <8.0.0", "@babel/core@^7.11.6", "@babel/core@^7.12.0", "@babel/core@^7.12.3", "@babel/core@^7.13.0", "@babel/core@^7.18.6", "@babel/core@^7.19.6", "@babel/core@^7.4.0 || ^8.0.0-0 <8.0.0": + version "7.22.10" + resolved "https://registry.npmjs.org/@babel/core/-/core-7.22.10.tgz" + integrity sha512-fTmqbbUBAwCcre6zPzNngvsI0aNrPZe77AeqvDxWM9Nm+04RrJ3CAmGHA9f7lJQY6ZMhRztNemy4uslDxTX4Qw== + dependencies: + "@ampproject/remapping" "^2.2.0" + "@babel/code-frame" "^7.22.10" + "@babel/generator" "^7.22.10" + "@babel/helper-compilation-targets" "^7.22.10" + "@babel/helper-module-transforms" "^7.22.9" + "@babel/helpers" "^7.22.10" + "@babel/parser" "^7.22.10" + "@babel/template" "^7.22.5" + "@babel/traverse" "^7.22.10" + "@babel/types" "^7.22.10" + convert-source-map "^1.7.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.2.2" + semver "^6.3.1" + "@babel/core@7.12.9": version "7.12.9" resolved "https://registry.npmjs.org/@babel/core/-/core-7.12.9.tgz" @@ -188,27 +209,6 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/core@^7.12.3", "@babel/core@^7.18.6", "@babel/core@^7.19.6": - version "7.22.10" - resolved "https://registry.npmjs.org/@babel/core/-/core-7.22.10.tgz" - integrity sha512-fTmqbbUBAwCcre6zPzNngvsI0aNrPZe77AeqvDxWM9Nm+04RrJ3CAmGHA9f7lJQY6ZMhRztNemy4uslDxTX4Qw== - dependencies: - "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.22.10" - "@babel/generator" "^7.22.10" - "@babel/helper-compilation-targets" "^7.22.10" - "@babel/helper-module-transforms" "^7.22.9" - "@babel/helpers" "^7.22.10" - "@babel/parser" "^7.22.10" - "@babel/template" "^7.22.5" - "@babel/traverse" "^7.22.10" - "@babel/types" "^7.22.10" - convert-source-map "^1.7.0" - debug "^4.1.0" - gensync "^1.0.0-beta.2" - json5 "^2.2.2" - semver "^6.3.1" - "@babel/generator@^7.12.5", "@babel/generator@^7.18.7", "@babel/generator@^7.22.10", "@babel/generator@^7.23.3": version "7.23.3" resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.23.3.tgz" @@ -331,16 +331,16 @@ dependencies: "@babel/types" "^7.22.5" -"@babel/helper-plugin-utils@7.10.4": - version "7.10.4" - resolved "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz" - integrity sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg== - "@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.18.6", "@babel/helper-plugin-utils@^7.22.5", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": version "7.22.5" resolved "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.22.5.tgz" integrity sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg== +"@babel/helper-plugin-utils@7.10.4": + version "7.10.4" + resolved "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz" + integrity sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg== + "@babel/helper-remap-async-to-generator@^7.22.5", "@babel/helper-remap-async-to-generator@^7.22.9": version "7.22.9" resolved "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.22.9.tgz" @@ -451,7 +451,7 @@ "@babel/helper-create-class-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" -"@babel/plugin-proposal-object-rest-spread@7.12.1", "@babel/plugin-proposal-object-rest-spread@^7.12.1": +"@babel/plugin-proposal-object-rest-spread@^7.12.1", "@babel/plugin-proposal-object-rest-spread@7.12.1": version "7.12.1" resolved "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz" integrity sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA== @@ -528,13 +528,6 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-jsx@7.12.1": - version "7.12.1" - resolved "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz" - integrity sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-jsx@^7.22.5": version "7.22.5" resolved "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.22.5.tgz" @@ -542,6 +535,13 @@ dependencies: "@babel/helper-plugin-utils" "^7.22.5" +"@babel/plugin-syntax-jsx@7.12.1": + version "7.12.1" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz" + integrity sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + "@babel/plugin-syntax-logical-assignment-operators@^7.10.4": version "7.10.4" resolved "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz" @@ -563,7 +563,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.4" -"@babel/plugin-syntax-object-rest-spread@7.8.3", "@babel/plugin-syntax-object-rest-spread@^7.8.0", "@babel/plugin-syntax-object-rest-spread@^7.8.3": +"@babel/plugin-syntax-object-rest-spread@^7.8.0", "@babel/plugin-syntax-object-rest-spread@^7.8.3", "@babel/plugin-syntax-object-rest-spread@7.8.3": version "7.8.3" resolved "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz" integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== @@ -1279,7 +1279,7 @@ "@docsearch/css" "3.5.1" algoliasearch "^4.0.0" -"@docusaurus/core@2.4.1": +"@docusaurus/core@^2.0.0-alpha.60 || ^2.0.0", "@docusaurus/core@2.4.1": version "2.4.1" resolved "https://registry.npmjs.org/@docusaurus/core/-/core-2.4.1.tgz" integrity sha512-SNsY7PshK3Ri7vtsLXVeAJGS50nJN3RgF836zkyUfAD01Fq+sAk5EwWgLw+nnm5KVNGDu7PRR2kRGDsWvqpo0g== @@ -1502,7 +1502,7 @@ "@docusaurus/utils-validation" "2.4.1" tslib "^2.4.0" -"@docusaurus/plugin-google-gtag@2.4.1", "@docusaurus/plugin-google-gtag@^2.4.1": +"@docusaurus/plugin-google-gtag@^2.4.1", "@docusaurus/plugin-google-gtag@2.4.1": version "2.4.1" resolved "https://registry.npmjs.org/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-2.4.1.tgz" integrity sha512-mKIefK+2kGTQBYvloNEKtDmnRD7bxHLsBcxgnbt4oZwzi2nxCGjPX6+9SQO2KCN5HZbNrYmGo5GJfMgoRvy6uA== @@ -1573,7 +1573,7 @@ "@docusaurus/theme-search-algolia" "2.4.1" "@docusaurus/types" "2.4.1" -"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -1671,7 +1671,7 @@ fs-extra "^10.1.0" tslib "^2.4.0" -"@docusaurus/types@2.4.1": +"@docusaurus/types@*", "@docusaurus/types@2.4.1": version "2.4.1" resolved "https://registry.npmjs.org/@docusaurus/types/-/types-2.4.1.tgz" integrity sha512-0R+cbhpMkhbRXX138UOc/2XZFF8hiZa6ooZAEEJFp5scytzCw4tC1gChMFXrpa3d2tYE6AX8IrOEpSonLmfQuQ== @@ -1857,16 +1857,16 @@ "@nodelib/fs.stat" "2.0.5" run-parallel "^1.1.9" -"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": - version "2.0.5" - resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" - integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== - "@nodelib/fs.stat@^1.1.2": version "1.1.3" resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz" integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw== +"@nodelib/fs.stat@^2.0.2", "@nodelib/fs.stat@2.0.5": + version "2.0.5" + resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" + integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== + "@nodelib/fs.walk@^1.2.3": version "1.2.8" resolved "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz" @@ -1975,7 +1975,7 @@ "@svgr/babel-plugin-transform-react-native-svg" "^6.5.1" "@svgr/babel-plugin-transform-svg-component" "^6.5.1" -"@svgr/core@^6.5.1": +"@svgr/core@*", "@svgr/core@^6.0.0", "@svgr/core@^6.5.1": version "6.5.1" resolved "https://registry.npmjs.org/@svgr/core/-/core-6.5.1.tgz" integrity sha512-/xdLSWxK5QkqG524ONSjvg3V/FkNyCv538OIBdQqPNaAta3AsXj/Bd2FbvR87yMbXO2hFSWiAe/Q6IkVPDw+mw== @@ -2248,7 +2248,7 @@ "@types/history" "^4.7.11" "@types/react" "*" -"@types/react@*": +"@types/react@*", "@types/react@>= 16.8.0 < 19.0.0": version "18.2.20" resolved "https://registry.npmjs.org/@types/react/-/react-18.2.20.tgz" integrity sha512-WKNtmsLWJM/3D5mG4U84cysVY31ivmyw85dE84fOCk5Hx78wezB/XEjVPWl2JTZ5FkEeaTJf+VgUAUn3PE7Isw== @@ -2329,7 +2329,7 @@ dependencies: "@types/yargs-parser" "*" -"@webassemblyjs/ast@1.11.6", "@webassemblyjs/ast@^1.11.5": +"@webassemblyjs/ast@^1.11.5", "@webassemblyjs/ast@1.11.6": version "1.11.6" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.6.tgz" integrity sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q== @@ -2430,7 +2430,7 @@ "@webassemblyjs/wasm-gen" "1.11.6" "@webassemblyjs/wasm-parser" "1.11.6" -"@webassemblyjs/wasm-parser@1.11.6", "@webassemblyjs/wasm-parser@^1.11.5": +"@webassemblyjs/wasm-parser@^1.11.5", "@webassemblyjs/wasm-parser@1.11.6": version "1.11.6" resolved "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.6.tgz" integrity sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ== @@ -2483,21 +2483,21 @@ acorn-walk@^8.0.0: resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== -acorn@^8.0.4, acorn@^8.7.1, acorn@^8.8.2: +acorn@^8, acorn@^8.0.4, acorn@^8.7.1, acorn@^8.8.2: version "8.10.0" resolved "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz" integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== -address@1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/address/-/address-1.1.2.tgz" - integrity sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA== - address@^1.0.1, address@^1.1.2: version "1.2.2" resolved "https://registry.npmjs.org/address/-/address-1.2.2.tgz" integrity sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA== +address@1.1.2: + version "1.1.2" + resolved "https://registry.npmjs.org/address/-/address-1.1.2.tgz" + integrity sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA== + aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz" @@ -2540,7 +2540,7 @@ ajv-keywords@^5.1.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5: +ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5, ajv@^6.9.1: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -2550,7 +2550,17 @@ ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.9.0: +ajv@^8.0.0: + version "8.12.0" + resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz" + integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + +ajv@^8.8.2, ajv@^8.9.0: version "8.12.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz" integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== @@ -2567,7 +2577,7 @@ algoliasearch-helper@^3.10.0: dependencies: "@algolia/events" "^4.0.1" -algoliasearch@^4.0.0, algoliasearch@^4.13.1: +algoliasearch@^4.0.0, algoliasearch@^4.13.1, "algoliasearch@>= 3.1 < 6", "algoliasearch@>= 4.9.1 < 6": version "4.19.1" resolved "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.19.1.tgz" integrity sha512-IJF5b93b2MgAzcE/tuzW0yOPnuUyRgGAtaPv5UUywXM8kzqfdwZTO4sPJBzoGz1eOy6H9uEchsJsBFTELZSu+g== @@ -2725,16 +2735,16 @@ array-find-index@^1.0.1: resolved "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz" integrity sha512-M1HQyIXcBGtVywBt8WVdim+lrNaK7VHp99Qt5pSNziXznKHViIBbXWtfRTpEFpF/c4FdfxNAsCCwPp5phBYJtw== -array-flatten@1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" - integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== - array-flatten@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz" integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ== +array-flatten@1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== + array-union@^1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz" @@ -2829,7 +2839,7 @@ asn1@~0.2.3: dependencies: safer-buffer "~2.1.0" -assert-plus@1.0.0, assert-plus@^1.0.0: +assert-plus@^1.0.0, assert-plus@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== @@ -3005,16 +3015,6 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -base16@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/base16/-/base16-1.0.0.tgz" - integrity sha512-pNdYkNPiJUnEhnfXV56+sQy8+AaPcG3POZAUnwr4EeqCUZFz4u2PePbo3e5Gj4ziYPCWGUZT9RHisvJKnwFuBQ== - -base64-js@^1.3.1: - version "1.5.1" - resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - base@^0.11.1: version "0.11.2" resolved "https://registry.npmjs.org/base/-/base-0.11.2.tgz" @@ -3028,6 +3028,16 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" +base16@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/base16/-/base16-1.0.0.tgz" + integrity sha512-pNdYkNPiJUnEhnfXV56+sQy8+AaPcG3POZAUnwr4EeqCUZFz4u2PePbo3e5Gj4ziYPCWGUZT9RHisvJKnwFuBQ== + +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + batch@0.6.1: version "0.6.1" resolved "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz" @@ -3140,7 +3150,7 @@ bluebird@~3.4.1: body-parser@1.20.2: version "1.20.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" + resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz" integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== dependencies: bytes "3.1.2" @@ -3240,17 +3250,7 @@ braces@^3.0.2, braces@~3.0.2: dependencies: fill-range "^7.0.1" -browserslist@4.14.2, browserslist@^4.12.0: - version "4.14.2" - resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.14.2.tgz" - integrity sha512-HI4lPveGKUR0x2StIz+2FXfDk9SfVMrxn6PLh1JeGUwcuoDkdKZebWiyLRJ68iIPDpMI4JLVDf7S7XzslgWOhw== - dependencies: - caniuse-lite "^1.0.30001125" - electron-to-chromium "^1.3.564" - escalade "^3.0.2" - node-releases "^1.1.61" - -browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.18.1, browserslist@^4.21.4, browserslist@^4.21.5, browserslist@^4.21.9: +browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.18.1, browserslist@^4.21.4, browserslist@^4.21.5, browserslist@^4.21.9, "browserslist@>= 4.21.0": version "4.21.10" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz" integrity sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ== @@ -3260,6 +3260,16 @@ browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.18.1, browserslist@^4 node-releases "^2.0.13" update-browserslist-db "^1.0.11" +browserslist@^4.12.0, browserslist@4.14.2: + version "4.14.2" + resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.14.2.tgz" + integrity sha512-HI4lPveGKUR0x2StIz+2FXfDk9SfVMrxn6PLh1JeGUwcuoDkdKZebWiyLRJ68iIPDpMI4JLVDf7S7XzslgWOhw== + dependencies: + caniuse-lite "^1.0.30001125" + electron-to-chromium "^1.3.564" + escalade "^3.0.2" + node-releases "^1.1.61" + buffer-alloc-unsafe@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz" @@ -3442,9 +3452,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001464, caniuse-lite@^1.0.30001517: - version "1.0.30001519" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001519.tgz" - integrity sha512-0QHgqR+Jv4bxHMp8kZ1Kn8CH55OikjKJ6JmKkZYP1F3D7w+lnFXF70nG5eNfsZS89jadi5Ywy5UCSKLAglIRkg== + version "1.0.30001629" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001629.tgz" + integrity sha512-c3dl911slnQhmxUIT4HhYzT7wnBK/XYpGnYLOj4nJBaRiw52Ibe7YxlDaAeRECvA786zCuExhxIUJ2K7nHMrBw== caseless@~0.12.0: version "0.12.0" @@ -3473,15 +3483,6 @@ chainsaw@~0.1.0: dependencies: traverse ">=0.3.0 <0.4" -chalk@2.4.2, chalk@^2.4.1, chalk@^2.4.2: - version "2.4.2" - resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - chalk@^1.0.0: version "1.1.3" resolved "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz" @@ -3493,6 +3494,24 @@ chalk@^1.0.0: strip-ansi "^3.0.0" supports-color "^2.0.0" +chalk@^2.4.1: + version "2.4.2" + resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chalk@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz" @@ -3509,6 +3528,15 @@ chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: ansi-styles "^4.1.0" supports-color "^7.1.0" +chalk@2.4.2: + version "2.4.2" + resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + character-entities-legacy@^1.0.0: version "1.1.4" resolved "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz" @@ -3536,6 +3564,19 @@ cheerio-select@^2.1.0: domhandler "^5.0.3" domutils "^3.0.1" +cheerio@^1.0.0-rc.12, cheerio@^1.0.0-rc.3: + version "1.0.0-rc.12" + resolved "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz" + integrity sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q== + dependencies: + cheerio-select "^2.1.0" + dom-serializer "^2.0.0" + domhandler "^5.0.3" + domutils "^3.0.1" + htmlparser2 "^8.0.1" + parse5 "^7.0.0" + parse5-htmlparser2-tree-adapter "^7.0.0" + cheerio@0.22.0: version "0.22.0" resolved "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz" @@ -3558,19 +3599,6 @@ cheerio@0.22.0: lodash.reject "^4.4.0" lodash.some "^4.4.0" -cheerio@^1.0.0-rc.12, cheerio@^1.0.0-rc.3: - version "1.0.0-rc.12" - resolved "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz" - integrity sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q== - dependencies: - cheerio-select "^2.1.0" - dom-serializer "^2.0.0" - domhandler "^5.0.3" - domutils "^3.0.1" - htmlparser2 "^8.0.1" - parse5 "^7.0.0" - parse5-htmlparser2-tree-adapter "^7.0.0" - chokidar@^3.4.2, chokidar@^3.5.3: version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" @@ -3661,13 +3689,6 @@ clone-deep@^4.0.1: kind-of "^6.0.2" shallow-clone "^3.0.0" -clone-response@1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz" - integrity sha512-yjLXh88P599UOyPTFX0POsd7WxnbsVsGohcwzHOLspIhhpalPw1BcqED8NblyZLKcGrL8dTgMlcaZxV2jAD41Q== - dependencies: - mimic-response "^1.0.0" - clone-response@^1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz" @@ -3675,6 +3696,13 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" +clone-response@1.0.2: + version "1.0.2" + resolved "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz" + integrity sha512-yjLXh88P599UOyPTFX0POsd7WxnbsVsGohcwzHOLspIhhpalPw1BcqED8NblyZLKcGrL8dTgMlcaZxV2jAD41Q== + dependencies: + mimic-response "^1.0.0" + clsx@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz" @@ -3721,16 +3749,16 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" - integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== - color-name@^1.0.0, color-name@~1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + color-string@^1.6.0, color-string@^1.9.0: version "1.9.1" resolved "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz" @@ -3787,7 +3815,17 @@ comma-separated-tokens@^1.0.0: resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz" integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== -commander@^2.19.0, commander@^2.20.0, commander@^2.8.1: +commander@^2.19.0: + version "2.20.3" + resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + +commander@^2.20.0: + version "2.20.3" + resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + +commander@^2.8.1: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== @@ -3909,7 +3947,7 @@ console-stream@^0.1.1: resolved "https://registry.npmjs.org/consolidated-events/-/consolidated-events-2.0.2.tgz" integrity sha512-2/uRVMdRypf5z/TW/ncD/66l75P5hH2vM/GR8Jf8HLc2xnfJtmina6F6du8+v4Z2vTrMo7jC+W1tmEEuuELgkQ== -content-disposition@0.5.2, content-disposition@^0.5.2: +content-disposition@^0.5.2, content-disposition@0.5.2: version "0.5.2" resolved "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz" integrity sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA== @@ -3943,7 +3981,7 @@ cookie-signature@1.0.6: cookie@0.6.0: version "0.6.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + resolved "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz" integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== copy-descriptor@^0.1.0: @@ -3990,16 +4028,16 @@ core-js@^3.23.3: resolved "https://registry.npmjs.org/core-js/-/core-js-3.32.0.tgz" integrity sha512-rd4rYZNlF3WuoYuRIDEmbR/ga9CeuWX9U05umAvgrrZoHY4Z++cp/xwPQMvUpBB4Ag6J8KfD80G0zwCyaSxDww== -core-util-is@1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" - integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== - core-util-is@~1.0.0: version "1.0.3" resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== + cosmiconfig@^5.0.0: version "5.2.1" resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.2.1.tgz" @@ -4049,15 +4087,6 @@ cross-fetch@^3.1.5: dependencies: node-fetch "^2.6.12" -cross-spawn@7.0.3, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== - dependencies: - path-key "^3.1.0" - shebang-command "^2.0.0" - which "^2.0.1" - cross-spawn@^5.0.1: version "5.1.0" resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz" @@ -4078,6 +4107,15 @@ cross-spawn@^6.0.0: shebang-command "^1.2.0" which "^1.2.9" +cross-spawn@^7.0.3, cross-spawn@7.0.3: + version "7.0.3" + resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + crowdin-cli@^0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/crowdin-cli/-/crowdin-cli-0.3.0.tgz" @@ -4092,7 +4130,7 @@ crypto-random-string@^2.0.0: resolved "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== -css-color-names@0.0.4, css-color-names@^0.0.4: +css-color-names@^0.0.4, css-color-names@0.0.4: version "0.0.4" resolved "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz" integrity sha512-zj5D7X1U2h2zsXOAM8EyUREBnnts6H+Jm+d1M2DbiQQcUtnqgQsMrdo8JW9R80YFUmIdBZeMu5wvYM7hcgWP/Q== @@ -4188,14 +4226,6 @@ css-selector-parser@^1.0.0: resolved "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-1.4.1.tgz" integrity sha512-HYPSb7y/Z7BNDCOrakL4raGO2zltZkbeXyAd6Tg9obzix6QhzxCotdBl6VT0Dv4vZfJGVz3WL/xaEI9Ly3ul0g== -css-tree@1.0.0-alpha.37: - version "1.0.0-alpha.37" - resolved "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz" - integrity sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg== - dependencies: - mdn-data "2.0.4" - source-map "^0.6.1" - css-tree@^1.1.2, css-tree@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/css-tree/-/css-tree-1.1.3.tgz" @@ -4204,10 +4234,13 @@ css-tree@^1.1.2, css-tree@^1.1.3: mdn-data "2.0.14" source-map "^0.6.1" -css-what@2.1: - version "2.1.3" - resolved "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz" - integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg== +css-tree@1.0.0-alpha.37: + version "1.0.0-alpha.37" + resolved "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz" + integrity sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg== + dependencies: + mdn-data "2.0.4" + source-map "^0.6.1" css-what@^3.2.1: version "3.4.2" @@ -4219,6 +4252,11 @@ css-what@^6.0.1, css-what@^6.1.0: resolved "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== +css-what@2.1: + version "2.1.3" + resolved "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz" + integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg== + cssesc@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz" @@ -4379,20 +4417,55 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0: +debug@^2.2.0: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== dependencies: ms "2.0.0" -debug@4, debug@^4.1.0, debug@^4.1.1: +debug@^2.3.3: + version "2.6.9" + resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +debug@^2.6.0: + version "2.6.9" + resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +debug@^3.1.0: + version "3.2.7" + resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +debug@^3.2.7: + version "3.2.7" + resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +debug@^4.1.0, debug@^4.1.1, debug@4: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" +debug@2.6.9: + version "2.6.9" + resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + debug@4.3.1: version "4.3.1" resolved "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz" @@ -4400,13 +4473,6 @@ debug@4.3.1: dependencies: ms "2.1.2" -debug@^3.1.0, debug@^3.2.7: - version "3.2.7" - resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== - dependencies: - ms "^2.1.1" - decamelize@^1.1.2: version "1.2.0" resolved "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz" @@ -4574,16 +4640,16 @@ delayed-stream@~1.0.0: resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -depd@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" - integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== - depd@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== +depd@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== + destroy@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" @@ -4606,7 +4672,7 @@ detect-node@^2.0.4: resolved "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz" integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== -detect-port-alt@1.1.6, detect-port-alt@^1.1.6: +detect-port-alt@^1.1.6, detect-port-alt@1.1.6: version "1.1.6" resolved "https://registry.npmjs.org/detect-port-alt/-/detect-port-alt-1.1.6.tgz" integrity sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q== @@ -4627,6 +4693,13 @@ diacritics-map@^0.1.0: resolved "https://registry.npmjs.org/diacritics-map/-/diacritics-map-0.1.0.tgz" integrity sha512-3omnDTYrGigU0i4cJjvaKwD52B8aoqyX/NEIkukFFkogBemsIbhSa1O414fpTp5nuszJG6lvQ5vBvDVNCbSsaQ== +dir-glob@^3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz" + integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== + dependencies: + path-type "^4.0.0" + dir-glob@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/dir-glob/-/dir-glob-2.0.0.tgz" @@ -4635,13 +4708,6 @@ dir-glob@2.0.0: arrify "^1.0.1" path-type "^3.0.0" -dir-glob@^3.0.1: - version "3.0.1" - resolved "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz" - integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== - dependencies: - path-type "^4.0.0" - direction@^1.0.0: version "1.0.4" resolved "https://registry.npmjs.org/direction/-/direction-1.0.4.tgz" @@ -4745,14 +4811,6 @@ dom-converter@^0.2.0: dependencies: utila "~0.4" -dom-serializer@0: - version "0.2.2" - resolved "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz" - integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g== - dependencies: - domelementtype "^2.0.1" - entities "^2.0.0" - dom-serializer@^1.0.1: version "1.4.1" resolved "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz" @@ -4779,7 +4837,15 @@ dom-serializer@~0.1.0: domelementtype "^1.3.0" entities "^1.1.1" -domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1: +dom-serializer@0: + version "0.2.2" + resolved "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz" + integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g== + dependencies: + domelementtype "^2.0.1" + entities "^2.0.0" + +domelementtype@^1.3.0, domelementtype@^1.3.1, domelementtype@1: version "1.3.1" resolved "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz" integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== @@ -4810,7 +4876,7 @@ domhandler@^5.0.2, domhandler@^5.0.3: dependencies: domelementtype "^2.3.0" -domutils@1.5.1, domutils@^1.5.1: +domutils@^1.5.1, domutils@1.5.1: version "1.5.1" resolved "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz" integrity sha512-gSu5Oi/I+3wDENBsOWBiRK1eoGxcywYSqg3rR960/+EfY0CF4EX1VPkgHOZ3WiS/Jg2DtliF6BhWcHlfpYUcGw== @@ -4894,6 +4960,11 @@ download@^7.1.0: p-event "^2.1.0" pify "^3.0.0" +duplexer@^0.1.1, duplexer@^0.1.2: + version "0.1.2" + resolved "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz" + integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== + duplexer2@~0.1.4: version "0.1.4" resolved "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz" @@ -4906,11 +4977,6 @@ duplexer3@^0.1.4: resolved "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz" integrity sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA== -duplexer@^0.1.1, duplexer@^0.1.2: - version "0.1.2" - resolved "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz" - integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== - eastasianwidth@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz" @@ -5025,7 +5091,7 @@ enzyme-shallow-equal@^1.0.1, enzyme-shallow-equal@^1.0.5: has "^1.0.3" object-is "^1.1.5" -enzyme@^3.10.0: +enzyme@^3.0.0, enzyme@^3.10.0: version "3.11.0" resolved "https://registry.npmjs.org/enzyme/-/enzyme-3.11.0.tgz" integrity sha512-Dw8/Gs4vRjxY6/6i9wU0V+utmQO9kvh9XLnz3LIudviOnVYDEe2ec+0k+NQoMamn1VrjKgCUOWj5jG/5M5M0Qw== @@ -5162,16 +5228,21 @@ escape-html@^1.0.3, escape-html@~1.0.3: resolved "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== -escape-string-regexp@2.0.0, escape-string-regexp@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz" - integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== - -escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: +escape-string-regexp@^1.0.2: version "1.0.5" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== + +escape-string-regexp@^2.0.0, escape-string-regexp@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz" + integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== + escape-string-regexp@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" @@ -5326,7 +5397,7 @@ expand-template@^2.0.3: express@^4.17.1, express@^4.17.3: version "4.19.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" + resolved "https://registry.npmjs.org/express/-/express-4.19.2.tgz" integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== dependencies: accepts "~1.3.8" @@ -5383,7 +5454,15 @@ extend-shallow@^2.0.1: dependencies: is-extendable "^0.1.0" -extend-shallow@^3.0.0, extend-shallow@^3.0.2: +extend-shallow@^3.0.0: + version "3.0.2" + resolved "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz" + integrity sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q== + dependencies: + assign-symbols "^1.0.0" + is-extendable "^1.0.1" + +extend-shallow@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz" integrity sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q== @@ -5410,7 +5489,7 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" -extsprintf@1.3.0, extsprintf@^1.2.0: +extsprintf@^1.2.0, extsprintf@1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz" integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== @@ -5542,7 +5621,7 @@ figures@^1.3.5: escape-string-regexp "^1.0.5" object-assign "^4.1.0" -file-loader@^6.2.0: +file-loader@*, file-loader@^6.2.0: version "6.2.0" resolved "https://registry.npmjs.org/file-loader/-/file-loader-6.2.0.tgz" integrity sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw== @@ -5550,11 +5629,6 @@ file-loader@^6.2.0: loader-utils "^2.0.0" schema-utils "^3.0.0" -file-type@5.2.0, file-type@^5.2.0: - version "5.2.0" - resolved "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz" - integrity sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ== - file-type@^10.4.0, file-type@^10.7.0: version "10.11.0" resolved "https://registry.npmjs.org/file-type/-/file-type-10.11.0.tgz" @@ -5570,6 +5644,11 @@ file-type@^4.2.0: resolved "https://registry.npmjs.org/file-type/-/file-type-4.4.0.tgz" integrity sha512-f2UbFQEk7LXgWpi5ntcO86OeA/cC80fuDDDaX/fZ2ZGel+AF7leRQqBBW1eJNiiQkrZlAoM6P+VYP5P6bOlDEQ== +file-type@^5.2.0: + version "5.2.0" + resolved "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz" + integrity sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ== + file-type@^6.1.0: version "6.2.0" resolved "https://registry.npmjs.org/file-type/-/file-type-6.2.0.tgz" @@ -5580,6 +5659,11 @@ file-type@^8.1.0: resolved "https://registry.npmjs.org/file-type/-/file-type-8.1.0.tgz" integrity sha512-qyQ0pzAy78gVoJsmYeNgl8uH8yKhr1lVhW7JbzJmnlRi0I4R2eEDEJZVKG8agpDnLpacwNbDhLNG/LMdxHD2YQ== +file-type@5.2.0: + version "5.2.0" + resolved "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz" + integrity sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ== + filename-reserved-regex@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz" @@ -5594,16 +5678,16 @@ filenamify@^2.0.0: strip-outer "^1.0.0" trim-repeated "^1.0.0" -filesize@6.1.0: - version "6.1.0" - resolved "https://registry.npmjs.org/filesize/-/filesize-6.1.0.tgz" - integrity sha512-LpCHtPQ3sFx67z+uh2HnSyWSLLu5Jxo21795uRDuar/EOuYWXib5EmPaGIBuSnRqH2IODiKA2k5re/K9OnN/Yg== - filesize@^8.0.6: version "8.0.7" resolved "https://registry.npmjs.org/filesize/-/filesize-8.0.7.tgz" integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ== +filesize@6.1.0: + version "6.1.0" + resolved "https://registry.npmjs.org/filesize/-/filesize-6.1.0.tgz" + integrity sha512-LpCHtPQ3sFx67z+uh2HnSyWSLLu5Jxo21795uRDuar/EOuYWXib5EmPaGIBuSnRqH2IODiKA2k5re/K9OnN/Yg== + fill-range@^2.1.0: version "2.2.4" resolved "https://registry.npmjs.org/fill-range/-/fill-range-2.2.4.tgz" @@ -5663,14 +5747,6 @@ find-cache-dir@^3.3.1: make-dir "^3.0.2" pkg-dir "^4.1.0" -find-up@4.1.0, find-up@^4.0.0: - version "4.1.0" - resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" - integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== - dependencies: - locate-path "^5.0.0" - path-exists "^4.0.0" - find-up@^1.0.0: version "1.1.2" resolved "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz" @@ -5686,6 +5762,14 @@ find-up@^3.0.0: dependencies: locate-path "^3.0.0" +find-up@^4.0.0, find-up@4.1.0: + version "4.1.0" + resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" + integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== + dependencies: + locate-path "^5.0.0" + path-exists "^4.0.0" + find-up@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz" @@ -5711,7 +5795,7 @@ flux@^4.0.1: follow-redirects@^1.0.0, follow-redirects@^1.14.7: version "1.15.6" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz" integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== for-each@^0.3.3: @@ -5731,19 +5815,6 @@ forever-agent@~0.6.1: resolved "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== -fork-ts-checker-webpack-plugin@4.1.6: - version "4.1.6" - resolved "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-4.1.6.tgz" - integrity sha512-DUxuQaKoqfNne8iikd14SAkh5uw4+8vNifp6gmA73yYNS6ywLIWSLD/n/mBzHQRpW3J7rbATEakmiA8JvkTyZw== - dependencies: - "@babel/code-frame" "^7.5.5" - chalk "^2.4.1" - micromatch "^3.1.10" - minimatch "^3.0.4" - semver "^5.6.0" - tapable "^1.0.0" - worker-rpc "^0.1.0" - fork-ts-checker-webpack-plugin@^6.5.0: version "6.5.3" resolved "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-6.5.3.tgz" @@ -5763,6 +5834,19 @@ fork-ts-checker-webpack-plugin@^6.5.0: semver "^7.3.2" tapable "^1.0.0" +fork-ts-checker-webpack-plugin@4.1.6: + version "4.1.6" + resolved "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-4.1.6.tgz" + integrity sha512-DUxuQaKoqfNne8iikd14SAkh5uw4+8vNifp6gmA73yYNS6ywLIWSLD/n/mBzHQRpW3J7rbATEakmiA8JvkTyZw== + dependencies: + "@babel/code-frame" "^7.5.5" + chalk "^2.4.1" + micromatch "^3.1.10" + minimatch "^3.0.4" + semver "^5.6.0" + tapable "^1.0.0" + worker-rpc "^0.1.0" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" @@ -5816,7 +5900,17 @@ fs-extra@^10.1.0: jsonfile "^6.0.1" universalify "^2.0.0" -fs-extra@^9.0.0, fs-extra@^9.0.1: +fs-extra@^9.0.0: + version "9.1.0" + resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz" + integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== + dependencies: + at-least-node "^1.0.0" + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + +fs-extra@^9.0.1: version "9.1.0" resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz" integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== @@ -5925,11 +6019,6 @@ get-stdin@^4.0.1: resolved "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz" integrity sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw== -get-stream@3.0.0, get-stream@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz" - integrity sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ== - get-stream@^2.2.0: version "2.3.1" resolved "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz" @@ -5938,6 +6027,11 @@ get-stream@^2.2.0: object-assign "^4.0.1" pinkie-promise "^2.0.0" +get-stream@^3.0.0, get-stream@3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz" + integrity sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ== + get-stream@^4.0.0, get-stream@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz" @@ -6060,7 +6154,7 @@ global-dirs@^3.0.0: dependencies: ini "2.0.0" -global-modules@2.0.0, global-modules@^2.0.0: +global-modules@^2.0.0, global-modules@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/global-modules/-/global-modules-2.0.0.tgz" integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== @@ -6088,18 +6182,6 @@ globalthis@^1.0.3: dependencies: define-properties "^1.1.3" -globby@11.0.1: - version "11.0.1" - resolved "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz" - integrity sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ== - dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" - slash "^3.0.0" - globby@^11.0.1, globby@^11.0.4, globby@^11.1.0: version "11.1.0" resolved "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz" @@ -6136,6 +6218,18 @@ globby@^8.0.1: pify "^3.0.0" slash "^1.0.0" +globby@11.0.1: + version "11.0.1" + resolved "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz" + integrity sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ== + dependencies: + array-union "^2.1.0" + dir-glob "^3.0.1" + fast-glob "^3.1.1" + ignore "^5.1.4" + merge2 "^1.3.0" + slash "^3.0.0" + globule@^1.0.0: version "1.3.4" resolved "https://registry.npmjs.org/globule/-/globule-1.3.4.tgz" @@ -6247,6 +6341,13 @@ gulp-header@^1.7.1: lodash.template "^4.4.0" through2 "^2.0.0" +gzip-size@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz" + integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q== + dependencies: + duplexer "^0.1.2" + gzip-size@5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/gzip-size/-/gzip-size-5.1.1.tgz" @@ -6255,13 +6356,6 @@ gzip-size@5.1.1: duplexer "^0.1.1" pify "^4.0.1" -gzip-size@^6.0.0: - version "6.0.0" - resolved "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz" - integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q== - dependencies: - duplexer "^0.1.2" - handle-thing@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz" @@ -6656,21 +6750,31 @@ htmlparser2@^8.0.1: domutils "^3.0.1" entities "^4.4.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - http-cache-semantics@^4.0.0: version "4.1.1" resolved "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + http-deceiver@^1.2.7: version "1.2.7" resolved "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz" integrity sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw== +http-errors@~1.6.2: + version "1.6.3" + resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" + integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A== + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.0" + statuses ">= 1.4.0 < 2" + http-errors@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz" @@ -6682,16 +6786,6 @@ http-errors@2.0.0: statuses "2.0.1" toidentifier "1.0.1" -http-errors@~1.6.2: - version "1.6.3" - resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" - integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A== - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.0" - statuses ">= 1.4.0 < 2" - http-parser-js@>=0.5.1: version "0.5.8" resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz" @@ -6817,16 +6911,16 @@ immediate@^3.2.3: resolved "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz" integrity sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q== -immer@8.0.1: - version "8.0.1" - resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz" - integrity sha512-aqXhGP7//Gui2+UrEtvxZxSquQVXTpZ7KDxfCcKAF3Vysvw0CViVaW9RZ1j1xlIYqaaaipBoqdqeibkc18PNvA== - immer@^9.0.7: version "9.0.21" resolved "https://registry.npmjs.org/immer/-/immer-9.0.21.tgz" integrity sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA== +immer@8.0.1: + version "8.0.1" + resolved "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz" + integrity sha512-aqXhGP7//Gui2+UrEtvxZxSquQVXTpZ7KDxfCcKAF3Vysvw0CViVaW9RZ1j1xlIYqaaaipBoqdqeibkc18PNvA== + import-fresh@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-2.0.0.tgz" @@ -6888,7 +6982,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.0, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3: +inherits@^2.0.0, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3, inherits@2, inherits@2.0.4: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -6898,16 +6992,16 @@ inherits@2.0.3: resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw== -ini@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.8" resolved "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== +ini@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + inline-style-parser@0.1.1: version "0.1.1" resolved "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz" @@ -6947,16 +7041,16 @@ ip-regex@^4.1.0: resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-4.3.0.tgz" integrity sha512-B9ZWJxHHOHUhUjCPrMpLD4xEq35bUTClHM1S6CBU5ixQnkZmwipwgc96vAd7AAGM9TGHvJR+Uss+/Ak6UphK+Q== -ipaddr.js@1.9.1: - version "1.9.1" - resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" - integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== - ipaddr.js@^2.0.1: version "2.1.0" resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz" integrity sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ== +ipaddr.js@1.9.1: + version "1.9.1" + resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== + is-absolute-url@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-2.1.0.tgz" @@ -6969,7 +7063,7 @@ is-accessor-descriptor@^1.0.1: dependencies: hasown "^2.0.0" -is-alphabetical@1.0.4, is-alphabetical@^1.0.0: +is-alphabetical@^1.0.0, is-alphabetical@1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz" integrity sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg== @@ -7246,7 +7340,12 @@ is-path-inside@^3.0.2: resolved "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz" integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== -is-plain-obj@^1.0.0, is-plain-obj@^1.1.0: +is-plain-obj@^1.0.0: + version "1.1.0" + resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz" + integrity sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg== + +is-plain-obj@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz" integrity sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg== @@ -7296,7 +7395,7 @@ is-retry-allowed@^1.0.0, is-retry-allowed@^1.1.0: resolved "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz" integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== -is-root@2.1.0, is-root@^2.1.0: +is-root@^2.1.0, is-root@2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/is-root/-/is-root-2.1.0.tgz" integrity sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg== @@ -7308,7 +7407,12 @@ is-shared-array-buffer@^1.0.2: dependencies: call-bind "^1.0.2" -is-stream@^1.0.0, is-stream@^1.1.0: +is-stream@^1.0.0: + version "1.1.0" + resolved "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz" + integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ== + +is-stream@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz" integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ== @@ -7409,21 +7513,26 @@ is2@^2.0.6: ip-regex "^4.1.0" is-url "^1.2.4" +isarray@^2.0.5: + version "2.0.5" + resolved "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz" + integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== + isarray@0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== -isarray@1.0.0, isarray@~1.0.0: +isarray@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== -isarray@^2.0.5: - version "2.0.5" - resolved "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz" - integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== - isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" @@ -7515,7 +7624,15 @@ jpegtran-bin@^4.0.0: resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== -js-yaml@^3.13.1, js-yaml@^3.8.1: +js-yaml@^3.13.1: + version "3.14.1" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +js-yaml@^3.8.1: version "3.14.1" resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz" integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== @@ -7604,13 +7721,6 @@ jsprim@^1.2.2: json-schema "0.4.0" verror "1.10.0" -keyv@3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/keyv/-/keyv-3.0.0.tgz" - integrity sha512-eguHnq22OE3uVoSYG0LVWNP+4ppamWr9+zWBe1bsNcovIMy6huUJFPgy4mGwCd/rnl3vOLGW1MTlu4c57CT1xA== - dependencies: - json-buffer "3.0.0" - keyv@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz" @@ -7618,7 +7728,28 @@ keyv@^3.0.0: dependencies: json-buffer "3.0.0" -kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: +keyv@3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/keyv/-/keyv-3.0.0.tgz" + integrity sha512-eguHnq22OE3uVoSYG0LVWNP+4ppamWr9+zWBe1bsNcovIMy6huUJFPgy4mGwCd/rnl3vOLGW1MTlu4c57CT1xA== + dependencies: + json-buffer "3.0.0" + +kind-of@^3.0.2: + version "3.2.2" + resolved "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + +kind-of@^3.0.3: + version "3.2.2" + resolved "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + +kind-of@^3.2.0: version "3.2.2" resolved "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz" integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== @@ -7715,15 +7846,6 @@ loader-runner@^4.2.0: resolved "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz" integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== -loader-utils@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz" - integrity sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ== - dependencies: - big.js "^5.2.2" - emojis-list "^3.0.0" - json5 "^2.1.2" - loader-utils@^2.0.0: version "2.0.4" resolved "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz" @@ -7738,6 +7860,15 @@ loader-utils@^3.2.0: resolved "https://registry.npmjs.org/loader-utils/-/loader-utils-3.2.1.tgz" integrity sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw== +loader-utils@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz" + integrity sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ== + dependencies: + big.js "^5.2.2" + emojis-list "^3.0.0" + json5 "^2.1.2" + locate-path@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz" @@ -7890,7 +8021,7 @@ lodash.templatesettings@^4.0.0: dependencies: lodash._reinterpolate "^3.0.0" -lodash.uniq@4.5.0, lodash.uniq@^4.5.0: +lodash.uniq@^4.5.0, lodash.uniq@4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz" integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== @@ -7935,11 +8066,6 @@ lower-case@^2.0.2: dependencies: tslib "^2.0.3" -lowercase-keys@1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz" - integrity sha512-RPlX0+PHuvxVDZ7xX+EBVAp4RsVxP/TdDSN2mJYdiq1Lc4Hz7EUSjUI7RZrKKlmrIzVhf6Jo2stj7++gVarS0A== - lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz" @@ -7950,6 +8076,11 @@ lowercase-keys@^2.0.0: resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz" integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== +lowercase-keys@1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz" + integrity sha512-RPlX0+PHuvxVDZ7xX+EBVAp4RsVxP/TdDSN2mJYdiq1Lc4Hz7EUSjUI7RZrKKlmrIzVhf6Jo2stj7++gVarS0A== + lpad-align@^1.0.1: version "1.1.2" resolved "https://registry.npmjs.org/lpad-align/-/lpad-align-1.1.2.tgz" @@ -7992,7 +8123,14 @@ lunr@^2.3.8: resolved "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz" integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== -make-dir@^1.0.0, make-dir@^1.2.0: +make-dir@^1.0.0: + version "1.3.0" + resolved "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz" + integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ== + dependencies: + pify "^3.0.0" + +make-dir@^1.2.0: version "1.3.0" resolved "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz" integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ== @@ -8192,24 +8330,57 @@ micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.2" picomatch "^2.3.1" -mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": - version "1.52.0" - resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" - integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== - mime-db@^1.28.0, mime-db@~1.33.0: version "1.33.0" resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz" integrity sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ== -mime-types@2.1.18, mime-types@^2.1.12, mime-types@~2.1.17: +"mime-db@>= 1.43.0 < 2": + version "1.52.0" + resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12, mime-types@~2.1.17, mime-types@2.1.18: version "2.1.18" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz" integrity sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ== dependencies: mime-db "~1.33.0" -mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.27: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@^2.1.31: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@~2.1.19: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@~2.1.24: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@~2.1.34: version "2.1.35" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -8248,14 +8419,7 @@ minimalistic-assert@^1.0.0: resolved "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz" integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== -minimatch@3.0.4: - version "3.0.4" - resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== - dependencies: - brace-expansion "^1.1.7" - -minimatch@3.1.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1: +minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -8269,6 +8433,13 @@ minimatch@~3.0.2: dependencies: brace-expansion "^1.1.7" +minimatch@3.0.4: + version "3.0.4" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6: version "1.2.8" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" @@ -8287,18 +8458,32 @@ mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: resolved "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -mkdirp@0.3.0: - version "0.3.0" - resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz" - integrity sha512-OHsdUcVAQ6pOtg5JYWpCBo9W/GySVuwvP9hueRMW7UqshC0tbfzLv8wjySTPm3tfUZ/21CE9E1pJagOA91Pxew== - -"mkdirp@>=0.5 0", mkdirp@^0.5.1, mkdirp@^0.5.6, mkdirp@~0.5.1: +mkdirp@^0.5.1, mkdirp@~0.5.1: version "0.5.6" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== dependencies: minimist "^1.2.6" +mkdirp@^0.5.6: + version "0.5.6" + resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" + integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== + dependencies: + minimist "^1.2.6" + +"mkdirp@>=0.5 0": + version "0.5.6" + resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" + integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== + dependencies: + minimist "^1.2.6" + +mkdirp@0.3.0: + version "0.3.0" + resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz" + integrity sha512-OHsdUcVAQ6pOtg5JYWpCBo9W/GySVuwvP9hueRMW7UqshC0tbfzLv8wjySTPm3tfUZ/21CE9E1pJagOA91Pxew== + moo@^0.5.0: version "0.5.2" resolved "https://registry.npmjs.org/moo/-/moo-0.5.2.tgz" @@ -8309,16 +8494,16 @@ mrmime@^1.0.0: resolved "https://registry.npmjs.org/mrmime/-/mrmime-1.0.1.tgz" integrity sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw== +ms@^2.1.1, ms@2.1.2: + version "2.1.2" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + ms@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== -ms@2.1.2, ms@^2.1.1: - version "2.1.2" - resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" - integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== - ms@2.1.3: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" @@ -8465,15 +8650,6 @@ normalize-range@^0.1.2: resolved "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz" integrity sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA== -normalize-url@2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-2.0.1.tgz" - integrity sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw== - dependencies: - prepend-http "^2.0.0" - query-string "^5.0.1" - sort-keys "^2.0.0" - normalize-url@^3.0.0: version "3.3.0" resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-3.3.0.tgz" @@ -8489,6 +8665,15 @@ normalize-url@^6.0.1: resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +normalize-url@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-2.0.1.tgz" + integrity sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw== + dependencies: + prepend-http "^2.0.0" + query-string "^5.0.1" + sort-keys "^2.0.0" + not@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/not/-/not-0.1.0.tgz" @@ -8521,7 +8706,7 @@ nprogress@^0.2.0: resolved "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz" integrity sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA== -nth-check@^1.0.2, nth-check@~1.0.1: +nth-check@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz" integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg== @@ -8535,6 +8720,13 @@ nth-check@^2.0.0, nth-check@^2.0.1: dependencies: boolbase "^1.0.0" +nth-check@~1.0.1: + version "1.0.2" + resolved "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz" + integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg== + dependencies: + boolbase "~1.0.0" + num2fraction@^1.2.2: version "1.2.2" resolved "https://registry.npmjs.org/num2fraction/-/num2fraction-1.2.2.tgz" @@ -8985,6 +9177,13 @@ path-parse@^1.0.7: resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== +path-to-regexp@^1.7.0: + version "1.8.0" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz" + integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== + dependencies: + isarray "0.0.1" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" @@ -8995,13 +9194,6 @@ path-to-regexp@2.2.1: resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-2.2.1.tgz" integrity sha512-gu9bD6Ta5bwGrrU8muHzVOBFFREpp2iRkVfhBJahwJ6p6Xw20SjT0MxLnwkjOibQmGSYhiUnf2FLe7k+jcFmGQ== -path-to-regexp@^1.7.0: - version "1.8.0" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz" - integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== - dependencies: - isarray "0.0.1" - path-type@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz" @@ -9048,7 +9240,17 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1: resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== -pify@^2.0.0, pify@^2.2.0, pify@^2.3.0: +pify@^2.0.0: + version "2.3.0" + resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" + integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== + +pify@^2.2.0: + version "2.3.0" + resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" + integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== + +pify@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== @@ -9094,7 +9296,7 @@ pkg-dir@^4.1.0: dependencies: find-up "^4.0.0" -pkg-up@3.1.0, pkg-up@^3.1.0: +pkg-up@^3.1.0, pkg-up@3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz" integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA== @@ -9656,15 +9858,7 @@ postcss-zindex@^5.1.0: resolved "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-5.1.0.tgz" integrity sha512-fgFMf0OtVSBR1va1JNHYgMxYk73yhn/qb4uQDq1DLGYolz8gHCyr/sesEuGUaYs58E3ZJRcpoGuPVoB7Meiq9A== -postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.23, postcss@^7.0.27, postcss@^7.0.32: - version "7.0.39" - resolved "https://registry.npmjs.org/postcss/-/postcss-7.0.39.tgz" - integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== - dependencies: - picocolors "^0.2.1" - source-map "^0.6.1" - -postcss@^8.3.11, postcss@^8.4.14, postcss@^8.4.17, postcss@^8.4.21: +"postcss@^7.0.0 || ^8.0.1", postcss@^8.0.9, postcss@^8.1.0, postcss@^8.2.15, postcss@^8.2.2, postcss@^8.3.11, postcss@^8.4.14, postcss@^8.4.16, postcss@^8.4.17, postcss@^8.4.21: version "8.4.31" resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz" integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== @@ -9673,6 +9867,14 @@ postcss@^8.3.11, postcss@^8.4.14, postcss@^8.4.17, postcss@^8.4.21: picocolors "^1.0.0" source-map-js "^1.0.2" +postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.23, postcss@^7.0.27, postcss@^7.0.32: + version "7.0.39" + resolved "https://registry.npmjs.org/postcss/-/postcss-7.0.39.tgz" + integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== + dependencies: + picocolors "^0.2.1" + source-map "^0.6.1" + prebuild-install@^7.1.1: version "7.1.1" resolved "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz" @@ -9741,14 +9943,6 @@ promise@^7.1.1: dependencies: asap "~2.0.3" -prompts@2.4.0: - version "2.4.0" - resolved "https://registry.npmjs.org/prompts/-/prompts-2.4.0.tgz" - integrity sha512-awZAKrk3vN6CroQukBL+R9051a4R3zCZBlJm/HBfrSZ8iTpYix3VX1vU4mveiLpiwmOJT4wokTF9m6HUk4KqWQ== - dependencies: - kleur "^3.0.3" - sisteransi "^1.0.5" - prompts@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz" @@ -9757,6 +9951,14 @@ prompts@^2.4.2: kleur "^3.0.3" sisteransi "^1.0.5" +prompts@2.4.0: + version "2.4.0" + resolved "https://registry.npmjs.org/prompts/-/prompts-2.4.0.tgz" + integrity sha512-awZAKrk3vN6CroQukBL+R9051a4R3zCZBlJm/HBfrSZ8iTpYix3VX1vU4mveiLpiwmOJT4wokTF9m6HUk4KqWQ== + dependencies: + kleur "^3.0.3" + sisteransi "^1.0.5" + prop-types-exact@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/prop-types-exact/-/prop-types-exact-1.2.0.tgz" @@ -9766,7 +9968,7 @@ prop-types-exact@^1.2.0: object.assign "^4.1.0" reflect.ownkeys "^0.2.0" -prop-types@^15.0.0, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: +prop-types@^15.0.0, prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1, prop-types@>=15: version "15.8.1" resolved "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -9818,7 +10020,12 @@ punycode@^1.3.2: resolved "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz" integrity sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ== -punycode@^2.1.0, punycode@^2.1.1: +punycode@^2.1.0: + version "2.3.1" + resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== + +punycode@^2.1.1: version "2.3.1" resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== @@ -9840,7 +10047,7 @@ q@^1.1.2: resolved "https://registry.npmjs.org/q/-/q-1.5.1.tgz" integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw== -qs@6.11.0, qs@^6.4.0: +qs@^6.4.0, qs@6.11.0: version "6.11.0" resolved "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz" integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== @@ -9914,25 +10121,20 @@ randombytes@^2.1.0: dependencies: safe-buffer "^5.1.0" -range-parser@1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" - integrity sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A== - -range-parser@^1.2.1, range-parser@~1.2.1: +range-parser@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.5.2: - version "2.5.2" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" - integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== - dependencies: - bytes "3.1.2" - http-errors "2.0.0" - iconv-lite "0.4.24" - unpipe "1.0.0" +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + +range-parser@1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" + integrity sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A== raw-body@~1.1.0: version "1.1.7" @@ -9942,7 +10144,17 @@ raw-body@~1.1.0: bytes "1" string_decoder "0.10" -rc@1.2.8, rc@^1.2.7, rc@^1.2.8: +raw-body@2.5.2: + version "2.5.2" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz" + integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== + dependencies: + bytes "3.1.2" + http-errors "2.0.0" + iconv-lite "0.4.24" + unpipe "1.0.0" + +rc@^1.2.7, rc@^1.2.8, rc@1.2.8: version "1.2.8" resolved "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== @@ -10022,7 +10234,16 @@ react-dev-utils@^12.0.1: strip-ansi "^6.0.1" text-table "^0.2.0" -react-dom@^16.8.4: +react-dom@*, "react-dom@^16.6.0 || ^17.0.0 || ^18.0.0", "react-dom@^16.8.4 || ^17", "react-dom@^16.8.4 || ^17.0.0", "react-dom@^17.0.0 || ^16.3.0 || ^15.5.4", react-dom@^17.0.2, "react-dom@>= 16.8.0 < 19.0.0": + version "17.0.2" + resolved "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz" + integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + scheduler "^0.20.2" + +react-dom@^16.0.0-0, react-dom@^16.8.4: version "16.14.0" resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.14.0.tgz" integrity sha512-1gCeQXDLoIqMgqD3IO2Ah9bnf0w9kzhwN5q4FGnHZ67hBm9yePzB5JJAIQCc8x3pFnNlwFq4RidZggNAAkzWWw== @@ -10032,15 +10253,6 @@ react-dom@^16.8.4: prop-types "^15.6.2" scheduler "^0.19.1" -react-dom@^17.0.2: - version "17.0.2" - resolved "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz" - integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - scheduler "^0.20.2" - react-error-overlay@^6.0.11, react-error-overlay@^6.0.9: version "6.0.11" resolved "https://registry.npmjs.org/react-error-overlay/-/react-error-overlay-6.0.11.tgz" @@ -10094,6 +10306,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" +react-loadable@*, "react-loadable@npm:@docusaurus/react-loadable@5.5.2": + version "5.5.2" + resolved "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz" + integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== + dependencies: + "@types/react" "*" + prop-types "^15.6.2" + react-router-config@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/react-router-config/-/react-router-config-5.1.1.tgz" @@ -10114,7 +10334,7 @@ react-router-dom@^5.3.3: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" -react-router@5.3.4, react-router@^5.3.3: +react-router@^5.3.3, react-router@>=5, react-router@5.3.4: version "5.3.4" resolved "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz" integrity sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA== @@ -10148,7 +10368,7 @@ react-textarea-autosize@^8.3.2: use-composed-ref "^1.3.0" use-latest "^1.2.1" -react-waypoint@^10.3.0: +react-waypoint@^10.3.0, react-waypoint@>=9.0.2: version "10.3.0" resolved "https://registry.npmjs.org/react-waypoint/-/react-waypoint-10.3.0.tgz" integrity sha512-iF1y2c1BsoXuEGz08NoahaLFIGI9gTUAAOKip96HUmylRT6DUtpgoBPjk/Y8dfcFVmfVDvUzWjNXpZyKTOV0SQ== @@ -10158,7 +10378,15 @@ react-waypoint@^10.3.0: prop-types "^15.0.0" react-is "^17.0.1 || ^18.0.0" -react@^16.8.4: +react@*, "react@^15.0.2 || ^16.0.0 || ^17.0.0", "react@^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0", "react@^16.13.1 || ^17.0.0", "react@^16.6.0 || ^17.0.0 || ^18.0.0", "react@^16.8.0 || ^17.0.0 || ^18.0.0", "react@^16.8.4 || ^17", "react@^16.8.4 || ^17.0.0", "react@^17.0.0 || ^16.3.0 || ^15.5.4", react@^17.0.2, "react@>= 16.8.0 < 19.0.0", react@>=0.14.9, react@>=0.14.x, react@>=15, react@17.0.2: + version "17.0.2" + resolved "https://registry.npmjs.org/react/-/react-17.0.2.tgz" + integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + +"react@^0.14 || ^15.0.0 || ^16.0.0-alpha", react@^16.0.0-0, react@^16.14.0, react@^16.8.4, "react@0.13.x || 0.14.x || ^15.0.0-0 || ^16.0.0-0": version "16.14.0" resolved "https://registry.npmjs.org/react/-/react-16.14.0.tgz" integrity sha512-0X2CImDkJGApiAlcf0ODKIneSwBPhqJawOa5wCtKbu7ZECrmS26NvtSILynQ66cgkT/RJ4LidJOc3bUESwmU8g== @@ -10167,14 +10395,6 @@ react@^16.8.4: object-assign "^4.1.1" prop-types "^15.6.2" -react@^17.0.2: - version "17.0.2" - resolved "https://registry.npmjs.org/react/-/react-17.0.2.tgz" - integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - read-pkg-up@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz" @@ -10192,7 +10412,59 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6: +readable-stream@^2.0.0: + version "2.3.8" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz" + integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readable-stream@^2.0.1: + version "2.3.8" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz" + integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readable-stream@^2.0.2: + version "2.3.8" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz" + integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readable-stream@^2.2.2: + version "2.3.8" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz" + integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readable-stream@^2.3.0, readable-stream@^2.3.5: version "2.3.8" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz" integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== @@ -10214,6 +10486,19 @@ readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@~2.3.6: + version "2.3.8" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz" + integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz" @@ -10233,13 +10518,6 @@ rechoir@^0.6.2: dependencies: resolve "^1.1.6" -recursive-readdir@2.2.2: - version "2.2.2" - resolved "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz" - integrity sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg== - dependencies: - minimatch "3.0.4" - recursive-readdir@^2.2.2: version "2.2.3" resolved "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz" @@ -10247,6 +10525,13 @@ recursive-readdir@^2.2.2: dependencies: minimatch "^3.0.5" +recursive-readdir@2.2.2: + version "2.2.2" + resolved "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz" + integrity sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg== + dependencies: + minimatch "3.0.4" + redent@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz" @@ -10528,7 +10813,7 @@ resolve@^1.1.6, resolve@^1.10.0, resolve@^1.14.2, resolve@^1.3.2: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -responselike@1.0.2, responselike@^1.0.2: +responselike@^1.0.2, responselike@1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz" integrity sha512-/Fpe5guzJk1gPqdJLJR5u7eG/gNY4nImjbRDaVWVMRhne55TCmj2i9Q+54PBRfatRC8v/rIiv9BN0pMd9OV5EQ== @@ -10560,7 +10845,7 @@ rgba-regex@^1.0.0: resolved "https://registry.npmjs.org/rgba-regex/-/rgba-regex-1.0.0.tgz" integrity sha512-zgn5OjNQXLUTdq8m17KdaicF6w89TZs8ZU8y0AYENIU6wG8GG6LLm0yLSiPY8DmaYmHdgRW8rnApjoT0fQRfMg== -rimraf@2, rimraf@^2.5.4: +rimraf@^2.5.4: version "2.7.1" resolved "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz" integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== @@ -10574,6 +10859,13 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" +rimraf@2: + version "2.7.1" + resolved "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz" + integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== + dependencies: + glob "^7.1.3" + rst-selector-parser@^2.2.3: version "2.2.3" resolved "https://registry.npmjs.org/rst-selector-parser/-/rst-selector-parser-2.2.3.tgz" @@ -10621,15 +10913,20 @@ safe-array-concat@^1.0.0, safe-array-concat@^1.0.1: has-symbols "^1.0.3" isarray "^2.0.5" -safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@>=5.1.0, safe-buffer@~5.2.0, safe-buffer@5.2.1: + version "5.2.1" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: - version "5.2.1" - resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" - integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +safe-buffer@5.1.2: + version "5.1.2" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== safe-json-parse@~1.0.1: version "1.0.1" @@ -10652,7 +10949,7 @@ safe-regex@^1.1.0: dependencies: ret "~0.1.10" -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +safer-buffer@^2.0.2, safer-buffer@^2.1.0, "safer-buffer@>= 2.1.2 < 3", safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -10678,15 +10975,6 @@ scheduler@^0.20.2: loose-envify "^1.1.0" object-assign "^4.1.1" -schema-utils@2.7.0: - version "2.7.0" - resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz" - integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A== - dependencies: - "@types/json-schema" "^7.0.4" - ajv "^6.12.2" - ajv-keywords "^3.4.1" - schema-utils@^2.6.5: version "2.7.1" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz" @@ -10696,7 +10984,25 @@ schema-utils@^2.6.5: ajv "^6.12.4" ajv-keywords "^3.5.2" -schema-utils@^3.0.0, schema-utils@^3.1.1, schema-utils@^3.2.0: +schema-utils@^3.0.0: + version "3.3.0" + resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== + dependencies: + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + +schema-utils@^3.1.1: + version "3.3.0" + resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== + dependencies: + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + +schema-utils@^3.2.0: version "3.3.0" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz" integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== @@ -10715,6 +11021,20 @@ schema-utils@^4.0.0: ajv-formats "^2.1.1" ajv-keywords "^5.1.0" +schema-utils@2.7.0: + version "2.7.0" + resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz" + integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A== + dependencies: + "@types/json-schema" "^7.0.4" + ajv "^6.12.2" + ajv-keywords "^3.4.1" + +"search-insights@>= 1 < 3": + version "2.7.0" + resolved "https://registry.npmjs.org/search-insights/-/search-insights-2.7.0.tgz" + integrity sha512-GLbVaGgzYEKMvuJbHRhLi1qoBFnjXZGZ6l4LxOYPCp4lI2jDRB3jPU9/XNhMwv6kvnA9slTreq6pvK+b3o3aqg== + section-matter@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz" @@ -10761,12 +11081,42 @@ semver-truncate@^1.1.2: dependencies: semver "^5.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.4.1, semver@^5.5.0, semver@^5.6.0, semver@^5.7.0, semver@^5.7.1: +semver@^5.3.0: version "5.7.2" resolved "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz" integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== -semver@^6.0.0, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: +semver@^5.4.1: + version "5.7.2" + resolved "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@^5.5.0: + version "5.7.2" + resolved "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@^5.6.0, semver@^5.7.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@^6.0.0: + version "6.3.1" + resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^6.2.0: + version "6.3.1" + resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^6.3.0: + version "6.3.1" + resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^6.3.1: version "6.3.1" resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== @@ -10778,6 +11128,11 @@ semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semve dependencies: lru-cache "^6.0.0" +"semver@2 || 3 || 4 || 5": + version "5.7.2" + resolved "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + send@0.18.0: version "0.18.0" resolved "https://registry.npmjs.org/send/-/send-0.18.0.tgz" @@ -10904,6 +11259,20 @@ shallowequal@^1.1.0: resolved "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz" integrity sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ== +sharp@*, sharp@^0.32.6: + version "0.32.6" + resolved "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz" + integrity sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w== + dependencies: + color "^4.2.3" + detect-libc "^2.0.2" + node-addon-api "^6.1.0" + prebuild-install "^7.1.1" + semver "^7.5.4" + simple-get "^4.0.1" + tar-fs "^3.0.4" + tunnel-agent "^0.6.0" + sharp@^0.30.7: version "0.30.7" resolved "https://registry.npmjs.org/sharp/-/sharp-0.30.7.tgz" @@ -10918,20 +11287,6 @@ sharp@^0.30.7: tar-fs "^2.1.1" tunnel-agent "^0.6.0" -sharp@^0.32.6: - version "0.32.6" - resolved "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz" - integrity sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w== - dependencies: - color "^4.2.3" - detect-libc "^2.0.2" - node-addon-api "^6.1.0" - prebuild-install "^7.1.1" - semver "^7.5.4" - simple-get "^4.0.1" - tar-fs "^3.0.4" - tunnel-agent "^0.6.0" - shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz" @@ -10956,16 +11311,16 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shell-quote@1.7.2: - version "1.7.2" - resolved "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz" - integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== - shell-quote@^1.7.3: version "1.8.1" resolved "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz" integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA== +shell-quote@1.7.2: + version "1.7.2" + resolved "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz" + integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== + shelljs@^0.8.4, shelljs@^0.8.5: version "0.8.5" resolved "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz" @@ -11153,7 +11508,12 @@ source-map-url@^0.4.0: resolved "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz" integrity sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw== -source-map@^0.5.0, source-map@^0.5.6: +source-map@^0.5.0: + version "0.5.7" + resolved "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz" + integrity sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ== + +source-map@^0.5.6: version "0.5.7" resolved "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz" integrity sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ== @@ -11271,16 +11631,16 @@ static-extend@^0.1.1: define-property "^0.2.5" object-copy "^0.1.0" -statuses@2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" - integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== - "statuses@>= 1.4.0 < 2": version "1.5.0" resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== + std-env@^3.0.1: version "3.3.3" resolved "https://registry.npmjs.org/std-env/-/std-env-3.3.3.tgz" @@ -11299,12 +11659,58 @@ strict-uri-encode@^1.0.0: resolved "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz" integrity sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ== +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +string_decoder@0.10: + version "0.10.31" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + integrity sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ== + string-template@~0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz" integrity sha512-Yptehjogou2xm4UJbxJ4CxgZx12HBfeystp0y3x7s4Dj32ltVVG1Gg8YhKjHZkHicuKpZX/ffilA8505VbUbpw== -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width@^1.0.2 || 2 || 3 || 4": + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.2: + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.2.0: + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.2.3: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -11349,25 +11755,6 @@ string.prototype.trimstart@^1.0.7: define-properties "^1.2.0" es-abstract "^1.22.1" -string_decoder@0.10: - version "0.10.31" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" - integrity sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ== - -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== - dependencies: - safe-buffer "~5.1.0" - stringify-object@^3.3.0: version "3.3.0" resolved "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz" @@ -11377,13 +11764,6 @@ stringify-object@^3.3.0: is-obj "^1.0.1" is-regexp "^1.0.0" -strip-ansi@6.0.0: - version "6.0.0" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== - dependencies: - ansi-regex "^5.0.0" - strip-ansi@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz" @@ -11405,6 +11785,13 @@ strip-ansi@^7.0.1: dependencies: ansi-regex "^6.0.1" +strip-ansi@6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz" + integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== + dependencies: + ansi-regex "^5.0.0" + strip-bom-string@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz" @@ -11468,7 +11855,7 @@ strnum@^1.0.5: resolved "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz" integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA== -style-to-object@0.3.0, style-to-object@^0.3.0: +style-to-object@^0.3.0, style-to-object@0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/style-to-object/-/style-to-object-0.3.0.tgz" integrity sha512-CzFnRRXhzWIdItT3OmF8SQfWyahHhjq3HwcMNCNLn+N7klOOqPjMeG/4JSu77D7ypZdGvSzvkrbyeTMizz2VrA== @@ -11528,7 +11915,26 @@ svg-parser@^2.0.4: resolved "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz" integrity sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ== -svgo@^1.0.0, svgo@^1.3.2: +svgo@^1.0.0: + version "1.3.2" + resolved "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz" + integrity sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw== + dependencies: + chalk "^2.4.1" + coa "^2.0.2" + css-select "^2.0.0" + css-select-base-adapter "^0.1.1" + css-tree "1.0.0-alpha.37" + csso "^4.0.2" + js-yaml "^3.13.1" + mkdirp "~0.5.1" + object.values "^1.1.0" + sax "~1.2.4" + stable "^0.1.8" + unquote "~1.1.1" + util.promisify "~1.0.0" + +svgo@^1.3.2: version "1.3.2" resolved "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz" integrity sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw== @@ -11664,11 +12070,16 @@ terser@^5.10.0, terser@^5.16.8: commander "^2.20.0" source-map-support "~0.5.20" -text-table@0.2.0, text-table@^0.2.0: +text-table@^0.2.0, text-table@0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== +through@^2.3.8: + version "2.3.8" + resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" + integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== + through2@^2.0.0: version "2.0.5" resolved "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz" @@ -11677,11 +12088,6 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through@^2.3.8: - version "2.3.8" - resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== - thunky@^1.0.2: version "1.1.0" resolved "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz" @@ -11944,6 +12350,11 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== +"typescript@>= 2.7": + version "5.1.6" + resolved "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz" + integrity sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA== + ua-parser-js@^1.0.35: version "1.0.35" resolved "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.35.tgz" @@ -11998,10 +12409,10 @@ unicode-property-aliases-ecmascript@^2.0.0: resolved "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz" integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w== -unified@9.2.0: - version "9.2.0" - resolved "https://registry.npmjs.org/unified/-/unified-9.2.0.tgz" - integrity sha512-vx2Z0vY+a3YoTj8+pttM3tiJHCwY5UFbYdiWrwBEbHmK8pvsPj2rtAX2BFfgXen8T39CJWblWRDT4L5WGXtDdg== +unified@^9.0.0, unified@^9.2.2: + version "9.2.2" + resolved "https://registry.npmjs.org/unified/-/unified-9.2.2.tgz" + integrity sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ== dependencies: bail "^1.0.0" extend "^3.0.0" @@ -12010,10 +12421,10 @@ unified@9.2.0: trough "^1.0.0" vfile "^4.0.0" -unified@^9.0.0, unified@^9.2.2: - version "9.2.2" - resolved "https://registry.npmjs.org/unified/-/unified-9.2.2.tgz" - integrity sha512-Sg7j110mtefBD+qunSLO1lqOEKdrwBFBrR6Qd8f4uwkhWNlbkaqwHse6e7QvD3AP/MNoJdEDLaf8OxYyoWgorQ== +unified@9.2.0: + version "9.2.0" + resolved "https://registry.npmjs.org/unified/-/unified-9.2.0.tgz" + integrity sha512-vx2Z0vY+a3YoTj8+pttM3tiJHCwY5UFbYdiWrwBEbHmK8pvsPj2rtAX2BFfgXen8T39CJWblWRDT4L5WGXtDdg== dependencies: bail "^1.0.0" extend "^3.0.0" @@ -12049,7 +12460,7 @@ unique-string@^2.0.0: dependencies: crypto-random-string "^2.0.0" -unist-builder@2.0.3, unist-builder@^2.0.0: +unist-builder@^2.0.0, unist-builder@2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/unist-builder/-/unist-builder-2.0.3.tgz" integrity sha512-f98yt5pnlMWlzP539tPc4grGMsFaQQlP/vM396b00jngsiINumNmsY8rkXjfoi1c6QaM8nQ3vaGDuoKWbe/1Uw== @@ -12105,7 +12516,7 @@ unist-util-visit-parents@^3.0.0: "@types/unist" "^2.0.0" unist-util-is "^4.0.0" -unist-util-visit@2.0.3, unist-util-visit@^2.0.0, unist-util-visit@^2.0.3: +unist-util-visit@^2.0.0, unist-util-visit@^2.0.3, unist-util-visit@2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz" integrity sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q== @@ -12119,7 +12530,7 @@ universalify@^2.0.0: resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz" integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== -unpipe@1.0.0, unpipe@~1.0.0: +unpipe@~1.0.0, unpipe@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== @@ -12278,7 +12689,12 @@ utils-merge@1.0.1: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== -uuid@^3.0.1, uuid@^3.3.2: +uuid@^3.0.1: + version "3.4.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + +uuid@^3.3.2: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== @@ -12407,7 +12823,7 @@ webpack-bundle-analyzer@^4.5.0: webpack-dev-middleware@^5.3.1: version "5.3.4" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz#eb7b39281cbce10e104eb2b8bf2b63fce49a3517" + resolved "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz" integrity sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q== dependencies: colorette "^2.0.10" @@ -12465,7 +12881,7 @@ webpack-sources@^3.2.2, webpack-sources@^3.2.3: resolved "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@^5.73.0: +"webpack@^4.0.0 || ^5.0.0", "webpack@^4.37.0 || ^5.0.0", webpack@^5.0.0, webpack@^5.1.0, webpack@^5.20.0, webpack@^5.73.0, "webpack@>= 4", webpack@>=2, "webpack@>=4.41.1 || 5.x", "webpack@3 || 4 || 5": version "5.88.2" resolved "https://registry.npmjs.org/webpack/-/webpack-5.88.2.tgz" integrity sha512-JmcgNZ1iKj+aiR0OvTYtWQqJwq37Pf683dY9bVORwVbUrDhLhdn/PlO2sHsFHPkj7sHNQF3JwaAkp49V+Sq1tQ== @@ -12505,7 +12921,7 @@ webpackbar@^5.0.2: pretty-time "^1.1.0" std-env "^3.0.1" -websocket-driver@>=0.5.1, websocket-driver@^0.7.4: +websocket-driver@^0.7.4, websocket-driver@>=0.5.1: version "0.7.4" resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz" integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg== @@ -12558,7 +12974,14 @@ which-typed-array@^1.1.11, which-typed-array@^1.1.13: gopd "^1.0.1" has-tostringtag "^1.0.0" -which@^1.2.9, which@^1.3.1: +which@^1.2.9: + version "1.3.1" + resolved "https://registry.npmjs.org/which/-/which-1.3.1.tgz" + integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== + dependencies: + isexe "^2.0.0" + +which@^1.3.1: version "1.3.1" resolved "https://registry.npmjs.org/which/-/which-1.3.1.tgz" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== diff --git a/enterprise/enterprise_callbacks/example_logging_api.py b/enterprise/enterprise_callbacks/example_logging_api.py index 57ea99a67..c3d3f5e63 100644 --- a/enterprise/enterprise_callbacks/example_logging_api.py +++ b/enterprise/enterprise_callbacks/example_logging_api.py @@ -18,10 +18,6 @@ async def log_event(request: Request): return {"message": "Request received successfully"} except Exception as e: - print(f"Error processing request: {str(e)}") - import traceback - - traceback.print_exc() raise HTTPException(status_code=500, detail="Internal Server Error") diff --git a/enterprise/enterprise_callbacks/generic_api_callback.py b/enterprise/enterprise_callbacks/generic_api_callback.py index cf1d22e8f..ba189b149 100644 --- a/enterprise/enterprise_callbacks/generic_api_callback.py +++ b/enterprise/enterprise_callbacks/generic_api_callback.py @@ -120,6 +120,5 @@ class GenericAPILogger: ) return response except Exception as e: - traceback.print_exc() - verbose_logger.debug(f"Generic - {str(e)}\n{traceback.format_exc()}") + verbose_logger.error(f"Generic - {str(e)}\n{traceback.format_exc()}") pass diff --git a/enterprise/enterprise_hooks/banned_keywords.py b/enterprise/enterprise_hooks/banned_keywords.py index acd390d79..4cf68b2fd 100644 --- a/enterprise/enterprise_hooks/banned_keywords.py +++ b/enterprise/enterprise_hooks/banned_keywords.py @@ -82,7 +82,7 @@ class _ENTERPRISE_BannedKeywords(CustomLogger): except HTTPException as e: raise e except Exception as e: - traceback.print_exc() + verbose_proxy_logger.error(traceback.format_exc()) async def async_post_call_success_hook( self, diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index cbc14d2c2..8e642a026 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -118,4 +118,4 @@ class _ENTERPRISE_BlockedUserList(CustomLogger): except HTTPException as e: raise e except Exception as e: - traceback.print_exc() + verbose_proxy_logger.error(traceback.format_exc()) diff --git a/enterprise/enterprise_hooks/llm_guard.py b/enterprise/enterprise_hooks/llm_guard.py index 3a15ca52b..9db10cf79 100644 --- a/enterprise/enterprise_hooks/llm_guard.py +++ b/enterprise/enterprise_hooks/llm_guard.py @@ -92,7 +92,7 @@ class _ENTERPRISE_LLMGuard(CustomLogger): }, ) except Exception as e: - traceback.print_exc() + verbose_proxy_logger.error(traceback.format_exc()) raise e def should_proceed(self, user_api_key_dict: UserAPIKeyAuth, data: dict) -> bool: diff --git a/enterprise/utils.py b/enterprise/utils.py index 90b14314c..b8f660927 100644 --- a/enterprise/utils.py +++ b/enterprise/utils.py @@ -1,5 +1,7 @@ # Enterprise Proxy Util Endpoints +from typing import Optional, List from litellm._logging import verbose_logger +from litellm.proxy.proxy_server import PrismaClient, HTTPException import collections from datetime import datetime @@ -19,27 +21,76 @@ async def get_spend_by_tags(start_date=None, end_date=None, prisma_client=None): return response -async def ui_get_spend_by_tags(start_date: str, end_date: str, prisma_client): - - sql_query = """ - SELECT - jsonb_array_elements_text(request_tags) AS individual_request_tag, - DATE(s."startTime") AS spend_date, - COUNT(*) AS log_count, - SUM(spend) AS total_spend - FROM "LiteLLM_SpendLogs" s - WHERE - DATE(s."startTime") >= $1::date - AND DATE(s."startTime") <= $2::date - GROUP BY individual_request_tag, spend_date - ORDER BY spend_date - LIMIT 100; +async def ui_get_spend_by_tags( + start_date: str, + end_date: str, + prisma_client: Optional[PrismaClient] = None, + tags_str: Optional[str] = None, +): """ - response = await prisma_client.db.query_raw( - sql_query, - start_date, - end_date, - ) + Should cover 2 cases: + 1. When user is getting spend for all_tags. "all_tags" in tags_list + 2. When user is getting spend for specific tags. + """ + + # tags_str is a list of strings csv of tags + # tags_str = tag1,tag2,tag3 + # convert to list if it's not None + tags_list: Optional[List[str]] = None + if tags_str is not None and len(tags_str) > 0: + tags_list = tags_str.split(",") + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + response = None + if tags_list is None or (isinstance(tags_list, list) and "all-tags" in tags_list): + # Get spend for all tags + sql_query = """ + SELECT + jsonb_array_elements_text(request_tags) AS individual_request_tag, + DATE(s."startTime") AS spend_date, + COUNT(*) AS log_count, + SUM(spend) AS total_spend + FROM "LiteLLM_SpendLogs" s + WHERE + DATE(s."startTime") >= $1::date + AND DATE(s."startTime") <= $2::date + GROUP BY individual_request_tag, spend_date + ORDER BY total_spend DESC; + """ + response = await prisma_client.db.query_raw( + sql_query, + start_date, + end_date, + ) + else: + # filter by tags list + sql_query = """ + SELECT + individual_request_tag, + COUNT(*) AS log_count, + SUM(spend) AS total_spend + FROM ( + SELECT + jsonb_array_elements_text(request_tags) AS individual_request_tag, + DATE(s."startTime") AS spend_date, + spend + FROM "LiteLLM_SpendLogs" s + WHERE + DATE(s."startTime") >= $1::date + AND DATE(s."startTime") <= $2::date + ) AS subquery + WHERE individual_request_tag = ANY($3::text[]) + GROUP BY individual_request_tag + ORDER BY total_spend DESC; + """ + response = await prisma_client.db.query_raw( + sql_query, + start_date, + end_date, + tags_list, + ) # print("tags - spend") # print(response) diff --git a/litellm/__init__.py b/litellm/__init__.py index 3c78c9b27..e92ae355e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -5,8 +5,15 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.* ### INIT VARIABLES ### import threading, requests, os from typing import Callable, List, Optional, Dict, Union, Any, Literal +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching import Cache -from litellm._logging import set_verbose, _turn_on_debug, verbose_logger, json_logs +from litellm._logging import ( + set_verbose, + _turn_on_debug, + verbose_logger, + json_logs, + _turn_on_json, +) from litellm.proxy._types import ( KeyManagementSystem, KeyManagementSettings, @@ -53,6 +60,8 @@ _async_failure_callback: List[Callable] = ( pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False +redact_messages_in_exceptions: Optional[bool] = False +store_audit_logs = False # Enterprise feature, allow users to see audit logs ## end of callbacks ############# email: Optional[str] = ( @@ -95,7 +104,9 @@ common_cloud_provider_auth_params: dict = { } use_client: bool = False ssl_verify: bool = True +ssl_certificate: Optional[str] = None disable_streaming_logging: bool = False +in_memory_llm_clients_cache: dict = {} ### GUARDRAILS ### llamaguard_model_name: Optional[str] = None openai_moderations_model_name: Optional[str] = None @@ -221,7 +232,9 @@ default_team_settings: Optional[List] = None max_user_budget: Optional[float] = None max_end_user_budget: Optional[float] = None #### RELIABILITY #### -request_timeout: Optional[float] = 6000 +request_timeout: float = 6000 +module_level_aclient = AsyncHTTPHandler(timeout=request_timeout) +module_level_client = HTTPHandler(timeout=request_timeout) num_retries: Optional[int] = None # per model endpoint default_fallbacks: Optional[List] = None fallbacks: Optional[List] = None @@ -298,6 +311,7 @@ api_base = None headers = None api_version = None organization = None +project = None config_path = None ####### COMPLETION MODELS ################### open_ai_chat_completion_models: List = [] @@ -695,6 +709,7 @@ all_embedding_models = ( openai_image_generation_models = ["dall-e-2", "dall-e-3"] from .timeout import timeout +from .cost_calculator import completion_cost from .utils import ( client, exception_type, @@ -704,7 +719,6 @@ from .utils import ( create_pretrained_tokenizer, create_tokenizer, cost_per_token, - completion_cost, supports_function_calling, supports_parallel_function_calling, supports_vision, @@ -754,7 +768,7 @@ from .llms.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.ollama_chat import OllamaChatConfig from .llms.maritalk import MaritTalkConfig -from .llms.bedrock_httpx import AmazonCohereChatConfig +from .llms.bedrock_httpx import AmazonCohereChatConfig, AmazonConverseConfig from .llms.bedrock import ( AmazonTitanConfig, AmazonAI21Config, @@ -772,7 +786,11 @@ from .llms.openai import ( MistralConfig, DeepInfraConfig, ) -from .llms.azure import AzureOpenAIConfig, AzureOpenAIError +from .llms.azure import ( + AzureOpenAIConfig, + AzureOpenAIError, + AzureOpenAIAssistantsAPIConfig, +) from .llms.watsonx import IBMWatsonXAIConfig from .main import * # type: ignore from .integrations import * @@ -792,8 +810,13 @@ from .exceptions import ( APIConnectionError, APIResponseValidationError, UnprocessableEntityError, + InternalServerError, + LITELLM_EXCEPTION_TYPES, ) from .budget_manager import BudgetManager from .proxy.proxy_cli import run_server from .router import Router from .assistants.main import * +from .batches.main import * +from .scheduler import * +from .cost_calculator import response_cost_calculator diff --git a/litellm/_logging.py b/litellm/_logging.py index 0759ad51e..ab7a08f97 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,5 +1,6 @@ import logging, os, json from logging import Formatter +import traceback set_verbose = False json_logs = bool(os.getenv("JSON_LOGS", False)) @@ -39,6 +40,23 @@ verbose_proxy_logger.addHandler(handler) verbose_logger.addHandler(handler) +def _turn_on_json(): + handler = logging.StreamHandler() + handler.setFormatter(JsonFormatter()) + + # Define a list of the loggers to update + loggers = [verbose_router_logger, verbose_proxy_logger, verbose_logger] + + # Iterate through each logger and update its handlers + for logger in loggers: + # Remove all existing handlers + for h in logger.handlers[:]: + logger.removeHandler(h) + + # Add the new handler + logger.addHandler(handler) + + def _turn_on_debug(): verbose_logger.setLevel(level=logging.DEBUG) # set package log to debug verbose_router_logger.setLevel(level=logging.DEBUG) # set router logs to debug diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index dc6f35642..f2edb403e 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -1,10 +1,18 @@ -import litellm, traceback +from datetime import datetime +import litellm from litellm.proxy._types import UserAPIKeyAuth from .types.services import ServiceTypes, ServiceLoggerPayload from .integrations.prometheus_services import PrometheusServicesLogger from .integrations.custom_logger import CustomLogger from datetime import timedelta -from typing import Union +from typing import Union, Optional, TYPE_CHECKING, Any + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span = _Span +else: + Span = Any class ServiceLogging(CustomLogger): @@ -40,7 +48,13 @@ class ServiceLogging(CustomLogger): self.mock_testing_sync_failure_hook += 1 async def async_service_success_hook( - self, service: ServiceTypes, duration: float, call_type: str + self, + service: ServiceTypes, + call_type: str, + duration: float, + parent_otel_span: Optional[Span] = None, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, ): """ - For counting if the redis, postgres call is successful @@ -61,12 +75,25 @@ class ServiceLogging(CustomLogger): payload=payload ) + from litellm.proxy.proxy_server import open_telemetry_logger + + if parent_otel_span is not None and open_telemetry_logger is not None: + await open_telemetry_logger.async_service_success_hook( + payload=payload, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + ) + async def async_service_failure_hook( self, service: ServiceTypes, duration: float, error: Union[str, Exception], call_type: str, + parent_otel_span: Optional[Span] = None, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, ): """ - For counting if the redis, postgres call is unsuccessful @@ -95,6 +122,16 @@ class ServiceLogging(CustomLogger): payload=payload ) + from litellm.proxy.proxy_server import open_telemetry_logger + + if parent_otel_span is not None and open_telemetry_logger is not None: + await open_telemetry_logger.async_service_failure_hook( + payload=payload, + parent_otel_span=parent_otel_span, + start_time=start_time, + end_time=end_time, + ) + async def async_post_call_failure_hook( self, original_exception: Exception, user_api_key_dict: UserAPIKeyAuth ): diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py index 25d2433d7..eff9adfb2 100644 --- a/litellm/assistants/main.py +++ b/litellm/assistants/main.py @@ -1,27 +1,83 @@ # What is this? ## Main file for assistants API logic from typing import Iterable -import os +from functools import partial +import os, asyncio, contextvars import litellm -from openai import OpenAI +from openai import OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI from litellm import client -from litellm.utils import supports_httpx_timeout +from litellm.utils import ( + supports_httpx_timeout, + exception_type, + get_llm_provider, + get_secret, +) from ..llms.openai import OpenAIAssistantsAPI +from ..llms.azure import AzureAssistantsAPI from ..types.llms.openai import * from ..types.router import * +from .utils import get_optional_params_add_message ####### ENVIRONMENT VARIABLES ################### openai_assistants_api = OpenAIAssistantsAPI() +azure_assistants_api = AzureAssistantsAPI() ### ASSISTANTS ### +async def aget_assistants( + custom_llm_provider: Literal["openai", "azure"], + client: Optional[AsyncOpenAI] = None, + **kwargs, +) -> AsyncCursorPage[Assistant]: + loop = asyncio.get_event_loop() + ### PASS ARGS TO GET ASSISTANTS ### + kwargs["aget_assistants"] = True + try: + # Use a partial function to pass your keyword arguments + func = partial(get_assistants, custom_llm_provider, client, **kwargs) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( # type: ignore + model="", custom_llm_provider=custom_llm_provider + ) # type: ignore + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + return response # type: ignore + except Exception as e: + raise exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={}, + extra_kwargs=kwargs, + ) + + def get_assistants( - custom_llm_provider: Literal["openai"], - client: Optional[OpenAI] = None, + custom_llm_provider: Literal["openai", "azure"], + client: Optional[Any] = None, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, **kwargs, ) -> SyncCursorPage[Assistant]: - optional_params = GenericLiteLLMParams(**kwargs) + aget_assistants: Optional[bool] = kwargs.pop("aget_assistants", None) + if aget_assistants is not None and not isinstance(aget_assistants, bool): + raise Exception( + "Invalid value passed in for aget_assistants. Only bool or None allowed" + ) + optional_params = GenericLiteLLMParams( + api_key=api_key, api_base=api_base, api_version=api_version, **kwargs + ) ### TIMEOUT LOGIC ### timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 @@ -60,6 +116,7 @@ def get_assistants( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) + response = openai_assistants_api.get_assistants( api_base=api_base, api_key=api_key, @@ -67,6 +124,43 @@ def get_assistants( max_retries=optional_params.max_retries, organization=organization, client=client, + aget_assistants=aget_assistants, # type: ignore + ) # type: ignore + elif custom_llm_provider == "azure": + api_base = ( + optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") + ) # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token: Optional[str] = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_assistants_api.get_assistants( + api_base=api_base, + api_key=api_key, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=optional_params.max_retries, + client=client, + aget_assistants=aget_assistants, # type: ignore ) else: raise litellm.exceptions.BadRequestError( @@ -87,8 +181,43 @@ def get_assistants( ### THREADS ### +async def acreate_thread( + custom_llm_provider: Literal["openai", "azure"], **kwargs +) -> Thread: + loop = asyncio.get_event_loop() + ### PASS ARGS TO GET ASSISTANTS ### + kwargs["acreate_thread"] = True + try: + # Use a partial function to pass your keyword arguments + func = partial(create_thread, custom_llm_provider, **kwargs) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( # type: ignore + model="", custom_llm_provider=custom_llm_provider + ) # type: ignore + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + return response # type: ignore + except Exception as e: + raise exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={}, + extra_kwargs=kwargs, + ) + + def create_thread( - custom_llm_provider: Literal["openai"], + custom_llm_provider: Literal["openai", "azure"], messages: Optional[Iterable[OpenAICreateThreadParamsMessage]] = None, metadata: Optional[dict] = None, tool_resources: Optional[OpenAICreateThreadParamsToolResources] = None, @@ -117,6 +246,7 @@ def create_thread( ) ``` """ + acreate_thread = kwargs.get("acreate_thread", None) optional_params = GenericLiteLLMParams(**kwargs) ### TIMEOUT LOGIC ### @@ -165,7 +295,49 @@ def create_thread( max_retries=optional_params.max_retries, organization=organization, client=client, + acreate_thread=acreate_thread, ) + elif custom_llm_provider == "azure": + api_base = ( + optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") + ) # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + if isinstance(client, OpenAI): + client = None # only pass client if it's AzureOpenAI + + response = azure_assistants_api.create_thread( + messages=messages, + metadata=metadata, + api_base=api_base, + api_key=api_key, + azure_ad_token=azure_ad_token, + api_version=api_version, + timeout=timeout, + max_retries=optional_params.max_retries, + client=client, + acreate_thread=acreate_thread, + ) # type :ignore else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'create_thread'. Only 'openai' is supported.".format( @@ -179,16 +351,55 @@ def create_thread( request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore ), ) - return response + return response # type: ignore + + +async def aget_thread( + custom_llm_provider: Literal["openai", "azure"], + thread_id: str, + client: Optional[AsyncOpenAI] = None, + **kwargs, +) -> Thread: + loop = asyncio.get_event_loop() + ### PASS ARGS TO GET ASSISTANTS ### + kwargs["aget_thread"] = True + try: + # Use a partial function to pass your keyword arguments + func = partial(get_thread, custom_llm_provider, thread_id, client, **kwargs) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( # type: ignore + model="", custom_llm_provider=custom_llm_provider + ) # type: ignore + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + return response # type: ignore + except Exception as e: + raise exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={}, + extra_kwargs=kwargs, + ) def get_thread( - custom_llm_provider: Literal["openai"], + custom_llm_provider: Literal["openai", "azure"], thread_id: str, - client: Optional[OpenAI] = None, + client=None, **kwargs, ) -> Thread: """Get the thread object, given a thread_id""" + aget_thread = kwargs.pop("aget_thread", None) optional_params = GenericLiteLLMParams(**kwargs) ### TIMEOUT LOGIC ### @@ -228,6 +439,7 @@ def get_thread( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) + response = openai_assistants_api.get_thread( thread_id=thread_id, api_base=api_base, @@ -236,6 +448,47 @@ def get_thread( max_retries=optional_params.max_retries, organization=organization, client=client, + aget_thread=aget_thread, + ) + elif custom_llm_provider == "azure": + api_base = ( + optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") + ) # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + if isinstance(client, OpenAI): + client = None # only pass client if it's AzureOpenAI + + response = azure_assistants_api.get_thread( + thread_id=thread_id, + api_base=api_base, + api_key=api_key, + azure_ad_token=azure_ad_token, + api_version=api_version, + timeout=timeout, + max_retries=optional_params.max_retries, + client=client, + aget_thread=aget_thread, ) else: raise litellm.exceptions.BadRequestError( @@ -250,28 +503,90 @@ def get_thread( request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore ), ) - return response + return response # type: ignore ### MESSAGES ### -def add_message( - custom_llm_provider: Literal["openai"], +async def a_add_message( + custom_llm_provider: Literal["openai", "azure"], thread_id: str, role: Literal["user", "assistant"], content: str, attachments: Optional[List[Attachment]] = None, metadata: Optional[dict] = None, - client: Optional[OpenAI] = None, + client=None, + **kwargs, +) -> OpenAIMessage: + loop = asyncio.get_event_loop() + ### PASS ARGS TO GET ASSISTANTS ### + kwargs["a_add_message"] = True + try: + # Use a partial function to pass your keyword arguments + func = partial( + add_message, + custom_llm_provider, + thread_id, + role, + content, + attachments, + metadata, + client, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( # type: ignore + model="", custom_llm_provider=custom_llm_provider + ) # type: ignore + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + # Call the synchronous function using run_in_executor + response = init_response + return response # type: ignore + except Exception as e: + raise exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={}, + extra_kwargs=kwargs, + ) + + +def add_message( + custom_llm_provider: Literal["openai", "azure"], + thread_id: str, + role: Literal["user", "assistant"], + content: str, + attachments: Optional[List[Attachment]] = None, + metadata: Optional[dict] = None, + client=None, **kwargs, ) -> OpenAIMessage: ### COMMON OBJECTS ### - message_data = MessageData( + a_add_message = kwargs.pop("a_add_message", None) + _message_data = MessageData( role=role, content=content, attachments=attachments, metadata=metadata ) optional_params = GenericLiteLLMParams(**kwargs) + message_data = get_optional_params_add_message( + role=_message_data["role"], + content=_message_data["content"], + attachments=_message_data["attachments"], + metadata=_message_data["metadata"], + custom_llm_provider=custom_llm_provider, + ) + ### TIMEOUT LOGIC ### timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 # set timeout for 10 minutes by default @@ -318,6 +633,45 @@ def add_message( max_retries=optional_params.max_retries, organization=organization, client=client, + a_add_message=a_add_message, + ) + elif custom_llm_provider == "azure": + api_base = ( + optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") + ) # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_assistants_api.add_message( + thread_id=thread_id, + message_data=message_data, + api_base=api_base, + api_key=api_key, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=optional_params.max_retries, + client=client, + a_add_message=a_add_message, ) else: raise litellm.exceptions.BadRequestError( @@ -333,15 +687,61 @@ def add_message( ), ) - return response + return response # type: ignore + + +async def aget_messages( + custom_llm_provider: Literal["openai", "azure"], + thread_id: str, + client: Optional[AsyncOpenAI] = None, + **kwargs, +) -> AsyncCursorPage[OpenAIMessage]: + loop = asyncio.get_event_loop() + ### PASS ARGS TO GET ASSISTANTS ### + kwargs["aget_messages"] = True + try: + # Use a partial function to pass your keyword arguments + func = partial( + get_messages, + custom_llm_provider, + thread_id, + client, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( # type: ignore + model="", custom_llm_provider=custom_llm_provider + ) # type: ignore + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + # Call the synchronous function using run_in_executor + response = init_response + return response # type: ignore + except Exception as e: + raise exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={}, + extra_kwargs=kwargs, + ) def get_messages( - custom_llm_provider: Literal["openai"], + custom_llm_provider: Literal["openai", "azure"], thread_id: str, - client: Optional[OpenAI] = None, + client: Optional[Any] = None, **kwargs, ) -> SyncCursorPage[OpenAIMessage]: + aget_messages = kwargs.pop("aget_messages", None) optional_params = GenericLiteLLMParams(**kwargs) ### TIMEOUT LOGIC ### @@ -389,6 +789,44 @@ def get_messages( max_retries=optional_params.max_retries, organization=organization, client=client, + aget_messages=aget_messages, + ) + elif custom_llm_provider == "azure": + api_base = ( + optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") + ) # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_assistants_api.get_messages( + thread_id=thread_id, + api_base=api_base, + api_key=api_key, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=optional_params.max_retries, + client=client, + aget_messages=aget_messages, ) else: raise litellm.exceptions.BadRequestError( @@ -404,14 +842,21 @@ def get_messages( ), ) - return response + return response # type: ignore ### RUNS ### +def arun_thread_stream( + *, + event_handler: Optional[AssistantEventHandler] = None, + **kwargs, +) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: + kwargs["arun_thread"] = True + return run_thread(stream=True, event_handler=event_handler, **kwargs) # type: ignore -def run_thread( - custom_llm_provider: Literal["openai"], +async def arun_thread( + custom_llm_provider: Literal["openai", "azure"], thread_id: str, assistant_id: str, additional_instructions: Optional[str] = None, @@ -420,10 +865,79 @@ def run_thread( model: Optional[str] = None, stream: Optional[bool] = None, tools: Optional[Iterable[AssistantToolParam]] = None, - client: Optional[OpenAI] = None, + client: Optional[Any] = None, + **kwargs, +) -> Run: + loop = asyncio.get_event_loop() + ### PASS ARGS TO GET ASSISTANTS ### + kwargs["arun_thread"] = True + try: + # Use a partial function to pass your keyword arguments + func = partial( + run_thread, + custom_llm_provider, + thread_id, + assistant_id, + additional_instructions, + instructions, + metadata, + model, + stream, + tools, + client, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( # type: ignore + model="", custom_llm_provider=custom_llm_provider + ) # type: ignore + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + # Call the synchronous function using run_in_executor + response = init_response + return response # type: ignore + except Exception as e: + raise exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs={}, + extra_kwargs=kwargs, + ) + + +def run_thread_stream( + *, + event_handler: Optional[AssistantEventHandler] = None, + **kwargs, +) -> AssistantStreamManager[AssistantEventHandler]: + return run_thread(stream=True, event_handler=event_handler, **kwargs) # type: ignore + + +def run_thread( + custom_llm_provider: Literal["openai", "azure"], + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str] = None, + instructions: Optional[str] = None, + metadata: Optional[dict] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + tools: Optional[Iterable[AssistantToolParam]] = None, + client: Optional[Any] = None, + event_handler: Optional[AssistantEventHandler] = None, # for stream=True calls **kwargs, ) -> Run: """Run a given thread + assistant.""" + arun_thread = kwargs.pop("arun_thread", None) optional_params = GenericLiteLLMParams(**kwargs) ### TIMEOUT LOGIC ### @@ -463,6 +977,7 @@ def run_thread( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) + response = openai_assistants_api.run_thread( thread_id=thread_id, assistant_id=assistant_id, @@ -478,7 +993,53 @@ def run_thread( max_retries=optional_params.max_retries, organization=organization, client=client, + arun_thread=arun_thread, + event_handler=event_handler, ) + elif custom_llm_provider == "azure": + api_base = ( + optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") + ) # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_assistants_api.run_thread( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + stream=stream, + tools=tools, + api_base=str(api_base) if api_base is not None else None, + api_key=str(api_key) if api_key is not None else None, + api_version=str(api_version) if api_version is not None else None, + azure_ad_token=str(azure_ad_token) if azure_ad_token is not None else None, + timeout=timeout, + max_retries=optional_params.max_retries, + client=client, + arun_thread=arun_thread, + ) # type: ignore else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'run_thread'. Only 'openai' is supported.".format( @@ -492,4 +1053,4 @@ def run_thread( request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore ), ) - return response + return response # type: ignore diff --git a/litellm/assistants/utils.py b/litellm/assistants/utils.py new file mode 100644 index 000000000..ca5a1293d --- /dev/null +++ b/litellm/assistants/utils.py @@ -0,0 +1,158 @@ +import litellm +from typing import Optional, Union +from ..types.llms.openai import * + + +def get_optional_params_add_message( + role: Optional[str], + content: Optional[ + Union[ + str, + List[ + Union[ + MessageContentTextObject, + MessageContentImageFileObject, + MessageContentImageURLObject, + ] + ], + ] + ], + attachments: Optional[List[Attachment]], + metadata: Optional[dict], + custom_llm_provider: str, + **kwargs, +): + """ + Azure doesn't support 'attachments' for creating a message + + Reference - https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message + """ + passed_params = locals() + custom_llm_provider = passed_params.pop("custom_llm_provider") + special_params = passed_params.pop("kwargs") + for k, v in special_params.items(): + passed_params[k] = v + + default_params = { + "role": None, + "content": None, + "attachments": None, + "metadata": None, + } + + non_default_params = { + k: v + for k, v in passed_params.items() + if (k in default_params and v != default_params[k]) + } + optional_params = {} + + ## raise exception if non-default value passed for non-openai/azure embedding calls + def _check_valid_arg(supported_params): + if len(non_default_params.keys()) > 0: + keys = list(non_default_params.keys()) + for k in keys: + if ( + litellm.drop_params is True and k not in supported_params + ): # drop the unsupported non-default values + non_default_params.pop(k, None) + elif k not in supported_params: + raise litellm.utils.UnsupportedParamsError( + status_code=500, + message="k={}, not supported by {}. Supported params={}. To drop it from the call, set `litellm.drop_params = True`.".format( + k, custom_llm_provider, supported_params + ), + ) + return non_default_params + + if custom_llm_provider == "openai": + optional_params = non_default_params + elif custom_llm_provider == "azure": + supported_params = ( + litellm.AzureOpenAIAssistantsAPIConfig().get_supported_openai_create_message_params() + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.AzureOpenAIAssistantsAPIConfig().map_openai_params_create_message_params( + non_default_params=non_default_params, optional_params=optional_params + ) + for k in passed_params.keys(): + if k not in default_params.keys(): + optional_params[k] = passed_params[k] + return optional_params + + +def get_optional_params_image_gen( + n: Optional[int] = None, + quality: Optional[str] = None, + response_format: Optional[str] = None, + size: Optional[str] = None, + style: Optional[str] = None, + user: Optional[str] = None, + custom_llm_provider: Optional[str] = None, + **kwargs, +): + # retrieve all parameters passed to the function + passed_params = locals() + custom_llm_provider = passed_params.pop("custom_llm_provider") + special_params = passed_params.pop("kwargs") + for k, v in special_params.items(): + passed_params[k] = v + + default_params = { + "n": None, + "quality": None, + "response_format": None, + "size": None, + "style": None, + "user": None, + } + + non_default_params = { + k: v + for k, v in passed_params.items() + if (k in default_params and v != default_params[k]) + } + optional_params = {} + + ## raise exception if non-default value passed for non-openai/azure embedding calls + def _check_valid_arg(supported_params): + if len(non_default_params.keys()) > 0: + keys = list(non_default_params.keys()) + for k in keys: + if ( + litellm.drop_params is True and k not in supported_params + ): # drop the unsupported non-default values + non_default_params.pop(k, None) + elif k not in supported_params: + raise UnsupportedParamsError( + status_code=500, + message=f"Setting user/encoding format is not supported by {custom_llm_provider}. To drop it from the call, set `litellm.drop_params = True`.", + ) + return non_default_params + + if ( + custom_llm_provider == "openai" + or custom_llm_provider == "azure" + or custom_llm_provider in litellm.openai_compatible_providers + ): + optional_params = non_default_params + elif custom_llm_provider == "bedrock": + supported_params = ["size"] + _check_valid_arg(supported_params=supported_params) + if size is not None: + width, height = size.split("x") + optional_params["width"] = int(width) + optional_params["height"] = int(height) + elif custom_llm_provider == "vertex_ai": + supported_params = ["n"] + """ + All params here: https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/imagegeneration?project=adroit-crow-413218 + """ + _check_valid_arg(supported_params=supported_params) + if n is not None: + optional_params["sampleCount"] = int(n) + + for k in passed_params.keys(): + if k not in default_params.keys(): + optional_params[k] = passed_params[k] + return optional_params diff --git a/litellm/batches/main.py b/litellm/batches/main.py new file mode 100644 index 000000000..4043606d5 --- /dev/null +++ b/litellm/batches/main.py @@ -0,0 +1,589 @@ +""" +Main File for Batches API implementation + +https://platform.openai.com/docs/api-reference/batch + +- create_batch() +- retrieve_batch() +- cancel_batch() +- list_batch() + +""" + +import os +import asyncio +from functools import partial +import contextvars +from typing import Literal, Optional, Dict, Coroutine, Any, Union +import httpx + +import litellm +from litellm import client +from litellm.utils import supports_httpx_timeout +from ..types.router import * +from ..llms.openai import OpenAIBatchesAPI, OpenAIFilesAPI +from ..types.llms.openai import ( + CreateBatchRequest, + RetrieveBatchRequest, + CancelBatchRequest, + CreateFileRequest, + FileTypes, + FileObject, + Batch, + FileContentRequest, + HttpxBinaryResponseContent, +) + +####### ENVIRONMENT VARIABLES ################### +openai_batches_instance = OpenAIBatchesAPI() +openai_files_instance = OpenAIFilesAPI() +################################################# + + +async def acreate_file( + file: FileTypes, + purpose: Literal["assistants", "batch", "fine-tune"], + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Coroutine[Any, Any, FileObject]: + """ + Async: Files are used to upload documents that can be used with features like Assistants, Fine-tuning, and Batch API. + + LiteLLM Equivalent of POST: POST https://api.openai.com/v1/files + """ + try: + loop = asyncio.get_event_loop() + kwargs["acreate_file"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + create_file, + file, + purpose, + custom_llm_provider, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + + return response + except Exception as e: + raise e + + +def create_file( + file: FileTypes, + purpose: Literal["assistants", "batch", "fine-tune"], + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Union[FileObject, Coroutine[Any, Any, FileObject]]: + """ + Files are used to upload documents that can be used with features like Assistants, Fine-tuning, and Batch API. + + LiteLLM Equivalent of POST: POST https://api.openai.com/v1/files + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _create_file_request = CreateFileRequest( + file=file, + purpose=purpose, + extra_headers=extra_headers, + extra_body=extra_body, + ) + + _is_async = kwargs.pop("acreate_file", False) is True + + response = openai_files_instance.create_file( + _is_async=_is_async, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + create_file_data=_create_file_request, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e + + +async def afile_content( + file_id: str, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Coroutine[Any, Any, HttpxBinaryResponseContent]: + """ + Async: Get file contents + + LiteLLM Equivalent of GET https://api.openai.com/v1/files + """ + try: + loop = asyncio.get_event_loop() + kwargs["afile_content"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + file_content, + file_id, + custom_llm_provider, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + + return response + except Exception as e: + raise e + + +def file_content( + file_id: str, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Union[HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]]: + """ + Returns the contents of the specified file. + + LiteLLM Equivalent of POST: POST https://api.openai.com/v1/files + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _file_content_request = FileContentRequest( + file_id=file_id, + extra_headers=extra_headers, + extra_body=extra_body, + ) + + _is_async = kwargs.pop("afile_content", False) is True + + response = openai_files_instance.file_content( + _is_async=_is_async, + file_content_request=_file_content_request, + api_base=api_base, + api_key=api_key, + timeout=timeout, + max_retries=optional_params.max_retries, + organization=organization, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e + + +async def acreate_batch( + completion_window: Literal["24h"], + endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"], + input_file_id: str, + custom_llm_provider: Literal["openai"] = "openai", + metadata: Optional[Dict[str, str]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Coroutine[Any, Any, Batch]: + """ + Async: Creates and executes a batch from an uploaded file of request + + LiteLLM Equivalent of POST: https://api.openai.com/v1/batches + """ + try: + loop = asyncio.get_event_loop() + kwargs["acreate_batch"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + create_batch, + completion_window, + endpoint, + input_file_id, + custom_llm_provider, + metadata, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + + return response + except Exception as e: + raise e + + +def create_batch( + completion_window: Literal["24h"], + endpoint: Literal["/v1/chat/completions", "/v1/embeddings"], + input_file_id: str, + custom_llm_provider: Literal["openai"] = "openai", + metadata: Optional[Dict[str, str]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Union[Batch, Coroutine[Any, Any, Batch]]: + """ + Creates and executes a batch from an uploaded file of request + + LiteLLM Equivalent of POST: https://api.openai.com/v1/batches + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _is_async = kwargs.pop("acreate_batch", False) is True + + _create_batch_request = CreateBatchRequest( + completion_window=completion_window, + endpoint=endpoint, + input_file_id=input_file_id, + metadata=metadata, + extra_headers=extra_headers, + extra_body=extra_body, + ) + + response = openai_batches_instance.create_batch( + api_base=api_base, + api_key=api_key, + organization=organization, + create_batch_data=_create_batch_request, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e + + +async def aretrieve_batch( + batch_id: str, + custom_llm_provider: Literal["openai"] = "openai", + metadata: Optional[Dict[str, str]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Coroutine[Any, Any, Batch]: + """ + Async: Retrieves a batch. + + LiteLLM Equivalent of GET https://api.openai.com/v1/batches/{batch_id} + """ + try: + loop = asyncio.get_event_loop() + kwargs["aretrieve_batch"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + retrieve_batch, + batch_id, + custom_llm_provider, + metadata, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + + return response + except Exception as e: + raise e + + +def retrieve_batch( + batch_id: str, + custom_llm_provider: Literal["openai"] = "openai", + metadata: Optional[Dict[str, str]] = None, + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Union[Batch, Coroutine[Any, Any, Batch]]: + """ + Retrieves a batch. + + LiteLLM Equivalent of GET https://api.openai.com/v1/batches/{batch_id} + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _retrieve_batch_request = RetrieveBatchRequest( + batch_id=batch_id, + extra_headers=extra_headers, + extra_body=extra_body, + ) + + _is_async = kwargs.pop("aretrieve_batch", False) is True + + response = openai_batches_instance.retrieve_batch( + _is_async=_is_async, + retrieve_batch_data=_retrieve_batch_request, + api_base=api_base, + api_key=api_key, + organization=organization, + timeout=timeout, + max_retries=optional_params.max_retries, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e + + +def cancel_batch(): + pass + + +def list_batch(): + pass + + +async def acancel_batch(): + pass + + +async def alist_batch(): + pass diff --git a/litellm/caching.py b/litellm/caching.py index c8c1736d8..497dd2371 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -26,6 +26,16 @@ def print_verbose(print_statement): pass +def _get_parent_otel_span_from_kwargs(kwargs: Optional[dict] = None): + try: + if kwargs is None: + return None + _metadata = kwargs.get("metadata") or {} + return _metadata.get("litellm_parent_otel_span") + except: + return None + + class BaseCache: def set_cache(self, key, value, **kwargs): raise NotImplementedError @@ -233,6 +243,9 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="increment_cache", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) return result @@ -246,6 +259,9 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="increment_cache", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) verbose_logger.error( @@ -253,7 +269,6 @@ class RedisCache(BaseCache): str(e), value, ) - traceback.print_exc() raise e async def async_scan_iter(self, pattern: str, count: int = 100) -> list: @@ -277,6 +292,8 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="async_scan_iter", + start_time=start_time, + end_time=end_time, ) ) # DO NOT SLOW DOWN CALL B/C OF THIS return keys @@ -291,6 +308,8 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="async_scan_iter", + start_time=start_time, + end_time=end_time, ) ) raise e @@ -304,7 +323,12 @@ class RedisCache(BaseCache): _duration = end_time - start_time asyncio.create_task( self.service_logger_obj.async_service_failure_hook( - service=ServiceTypes.REDIS, duration=_duration, error=e + service=ServiceTypes.REDIS, + duration=_duration, + error=e, + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) # NON blocking - notify users Redis is throwing an exception @@ -313,7 +337,6 @@ class RedisCache(BaseCache): str(e), value, ) - traceback.print_exc() key = self.check_and_fix_namespace(key=key) async with _redis_client as redis_client: @@ -333,6 +356,9 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="async_set_cache", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) except Exception as e: @@ -344,6 +370,9 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="async_set_cache", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) # NON blocking - notify users Redis is throwing an exception @@ -352,9 +381,8 @@ class RedisCache(BaseCache): str(e), value, ) - traceback.print_exc() - async def async_set_cache_pipeline(self, cache_list, ttl=None): + async def async_set_cache_pipeline(self, cache_list, ttl=None, **kwargs): """ Use Redis Pipelines for bulk write operations """ @@ -392,6 +420,9 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="async_set_cache_pipeline", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) return results @@ -405,6 +436,9 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="async_set_cache_pipeline", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) @@ -413,7 +447,6 @@ class RedisCache(BaseCache): str(e), cache_value, ) - traceback.print_exc() async def batch_cache_write(self, key, value, **kwargs): print_verbose( @@ -438,6 +471,9 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="async_increment", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) return result @@ -451,6 +487,9 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="async_increment", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) verbose_logger.error( @@ -458,7 +497,6 @@ class RedisCache(BaseCache): str(e), value, ) - traceback.print_exc() raise e async def flush_cache_buffer(self): @@ -495,8 +533,9 @@ class RedisCache(BaseCache): return self._get_cache_logic(cached_response=cached_response) except Exception as e: # NON blocking - notify users Redis is throwing an exception - traceback.print_exc() - logging.debug("LiteLLM Caching: get() - Got exception from REDIS: ", e) + verbose_logger.error( + "LiteLLM Caching: get() - Got exception from REDIS: ", e + ) def batch_get_cache(self, key_list) -> dict: """ @@ -544,6 +583,9 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="async_get_cache", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) return response @@ -557,6 +599,9 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="async_get_cache", + start_time=start_time, + end_time=end_time, + parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) # NON blocking - notify users Redis is throwing an exception @@ -587,6 +632,8 @@ class RedisCache(BaseCache): service=ServiceTypes.REDIS, duration=_duration, call_type="async_batch_get_cache", + start_time=start_time, + end_time=end_time, ) ) @@ -612,6 +659,8 @@ class RedisCache(BaseCache): duration=_duration, error=e, call_type="async_batch_get_cache", + start_time=start_time, + end_time=end_time, ) ) print_verbose(f"Error occurred in pipeline read - {str(e)}") @@ -646,10 +695,9 @@ class RedisCache(BaseCache): error=e, call_type="sync_ping", ) - print_verbose( + verbose_logger.error( f"LiteLLM Redis Cache PING: - Got exception from REDIS : {str(e)}" ) - traceback.print_exc() raise e async def ping(self) -> bool: @@ -683,10 +731,9 @@ class RedisCache(BaseCache): call_type="async_ping", ) ) - print_verbose( + verbose_logger.error( f"LiteLLM Redis Cache PING: - Got exception from REDIS : {str(e)}" ) - traceback.print_exc() raise e async def delete_cache_keys(self, keys): @@ -1138,22 +1185,23 @@ class S3Cache(BaseCache): cached_response = ast.literal_eval(cached_response) if type(cached_response) is not dict: cached_response = dict(cached_response) - print_verbose( + verbose_logger.debug( f"Got S3 Cache: key: {key}, cached_response {cached_response}. Type Response {type(cached_response)}" ) return cached_response except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "NoSuchKey": - print_verbose( + verbose_logger.error( f"S3 Cache: The specified key '{key}' does not exist in the S3 bucket." ) return None except Exception as e: # NON blocking - notify users S3 is throwing an exception - traceback.print_exc() - print_verbose(f"S3 Caching: get_cache() - Got exception from S3: {e}") + verbose_logger.error( + f"S3 Caching: get_cache() - Got exception from S3: {e}" + ) async def async_get_cache(self, key, **kwargs): return self.get_cache(key=key, **kwargs) @@ -1234,8 +1282,7 @@ class DualCache(BaseCache): return result except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") raise e def get_cache(self, key, local_only: bool = False, **kwargs): @@ -1262,7 +1309,7 @@ class DualCache(BaseCache): print_verbose(f"get cache: cache result: {result}") return result except Exception as e: - traceback.print_exc() + verbose_logger.error(traceback.format_exc()) def batch_get_cache(self, keys: list, local_only: bool = False, **kwargs): try: @@ -1295,7 +1342,7 @@ class DualCache(BaseCache): print_verbose(f"async batch get cache: cache result: {result}") return result except Exception as e: - traceback.print_exc() + verbose_logger.error(traceback.format_exc()) async def async_get_cache(self, key, local_only: bool = False, **kwargs): # Try to fetch from in-memory cache first @@ -1328,7 +1375,7 @@ class DualCache(BaseCache): print_verbose(f"get cache: cache result: {result}") return result except Exception as e: - traceback.print_exc() + verbose_logger.error(traceback.format_exc()) async def async_batch_get_cache( self, keys: list, local_only: bool = False, **kwargs @@ -1368,7 +1415,7 @@ class DualCache(BaseCache): return result except Exception as e: - traceback.print_exc() + verbose_logger.error(traceback.format_exc()) async def async_set_cache(self, key, value, local_only: bool = False, **kwargs): print_verbose( @@ -1381,8 +1428,8 @@ class DualCache(BaseCache): if self.redis_cache is not None and local_only == False: await self.redis_cache.async_set_cache(key, value, **kwargs) except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") + verbose_logger.debug(traceback.format_exc()) async def async_batch_set_cache( self, cache_list: list, local_only: bool = False, **kwargs @@ -1401,11 +1448,11 @@ class DualCache(BaseCache): if self.redis_cache is not None and local_only == False: await self.redis_cache.async_set_cache_pipeline( - cache_list=cache_list, ttl=kwargs.get("ttl", None) + cache_list=cache_list, ttl=kwargs.get("ttl", None), **kwargs ) except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") + verbose_logger.debug(traceback.format_exc()) async def async_increment_cache( self, key, value: float, local_only: bool = False, **kwargs @@ -1429,8 +1476,8 @@ class DualCache(BaseCache): return result except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton async add_cache: {str(e)}") + verbose_logger.debug(traceback.format_exc()) raise e def flush_cache(self): @@ -1846,8 +1893,8 @@ class Cache: ) self.cache.set_cache(cache_key, cached_data, **kwargs) except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton add_cache: {str(e)}") + verbose_logger.debug(traceback.format_exc()) pass async def async_add_cache(self, result, *args, **kwargs): @@ -1864,8 +1911,8 @@ class Cache: ) await self.cache.async_set_cache(cache_key, cached_data, **kwargs) except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton add_cache: {str(e)}") + verbose_logger.debug(traceback.format_exc()) async def async_add_cache_pipeline(self, result, *args, **kwargs): """ @@ -1897,8 +1944,8 @@ class Cache: ) await asyncio.gather(*tasks) except Exception as e: - print_verbose(f"LiteLLM Cache: Excepton add_cache: {str(e)}") - traceback.print_exc() + verbose_logger.error(f"LiteLLM Cache: Excepton add_cache: {str(e)}") + verbose_logger.debug(traceback.format_exc()) async def batch_cache_write(self, result, *args, **kwargs): cache_key, cached_data, kwargs = self._add_cache_logic( diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py new file mode 100644 index 000000000..d1e2dab52 --- /dev/null +++ b/litellm/cost_calculator.py @@ -0,0 +1,352 @@ +# What is this? +## File for 'response_cost' calculation in Logging +from typing import Optional, Union, Literal, List +import litellm._logging +from litellm.utils import ( + ModelResponse, + EmbeddingResponse, + ImageResponse, + TranscriptionResponse, + TextCompletionResponse, + CallTypes, + cost_per_token, + print_verbose, + CostPerToken, + token_counter, +) +import litellm +from litellm import verbose_logger + + +# Extract the number of billion parameters from the model name +# only used for together_computer LLMs +def get_model_params_and_category(model_name) -> str: + """ + Helper function for calculating together ai pricing. + + Returns + - str - model pricing category if mapped else received model name + """ + import re + + model_name = model_name.lower() + re_params_match = re.search( + r"(\d+b)", model_name + ) # catch all decimals like 3b, 70b, etc + category = None + if re_params_match is not None: + params_match = str(re_params_match.group(1)) + params_match = params_match.replace("b", "") + if params_match is not None: + params_billion = float(params_match) + else: + return model_name + # Determine the category based on the number of parameters + if params_billion <= 4.0: + category = "together-ai-up-to-4b" + elif params_billion <= 8.0: + category = "together-ai-4.1b-8b" + elif params_billion <= 21.0: + category = "together-ai-8.1b-21b" + elif params_billion <= 41.0: + category = "together-ai-21.1b-41b" + elif params_billion <= 80.0: + category = "together-ai-41.1b-80b" + elif params_billion <= 110.0: + category = "together-ai-81.1b-110b" + if category is not None: + return category + + return model_name + + +def get_replicate_completion_pricing(completion_response=None, total_time=0.0): + # see https://replicate.com/pricing + # for all litellm currently supported LLMs, almost all requests go to a100_80gb + a100_80gb_price_per_second_public = ( + 0.001400 # assume all calls sent to A100 80GB for now + ) + if total_time == 0.0: # total time is in ms + start_time = completion_response["created"] + end_time = getattr(completion_response, "ended", time.time()) + total_time = end_time - start_time + + return a100_80gb_price_per_second_public * total_time / 1000 + + +def completion_cost( + completion_response=None, + model: Optional[str] = None, + prompt="", + messages: List = [], + completion="", + total_time=0.0, # used for replicate, sagemaker + call_type: Literal[ + "embedding", + "aembedding", + "completion", + "acompletion", + "atext_completion", + "text_completion", + "image_generation", + "aimage_generation", + "moderation", + "amoderation", + "atranscription", + "transcription", + "aspeech", + "speech", + ] = "completion", + ### REGION ### + custom_llm_provider=None, + region_name=None, # used for bedrock pricing + ### IMAGE GEN ### + size=None, + quality=None, + n=None, # number of images + ### CUSTOM PRICING ### + custom_cost_per_token: Optional[CostPerToken] = None, + custom_cost_per_second: Optional[float] = None, +) -> float: + """ + Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. + + Parameters: + completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request. + + [OPTIONAL PARAMS] + model (str): Optional. The name of the language model used in the completion calls + prompt (str): Optional. The input prompt passed to the llm + completion (str): Optional. The output completion text from the llm + total_time (float): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds + custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call. + custom_cost_per_second: Optional[float]: the cost per second for the llm api call. + + Returns: + float: The cost in USD dollars for the completion based on the provided parameters. + + Exceptions: + Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json + + + Note: + - If completion_response is provided, the function extracts token information and the model name from it. + - If completion_response is not provided, the function calculates token counts based on the model and input text. + - The cost is calculated based on the model, prompt tokens, and completion tokens. + - For certain models containing "togethercomputer" in the name, prices are based on the model size. + - For un-mapped Replicate models, the cost is calculated based on the total time used for the request. + """ + try: + if ( + (call_type == "aimage_generation" or call_type == "image_generation") + and model is not None + and isinstance(model, str) + and len(model) == 0 + and custom_llm_provider == "azure" + ): + model = "dall-e-2" # for dall-e-2, azure expects an empty model name + # Handle Inputs to completion_cost + prompt_tokens = 0 + completion_tokens = 0 + custom_llm_provider = None + if completion_response is not None: + # get input/output tokens from completion_response + prompt_tokens = completion_response.get("usage", {}).get("prompt_tokens", 0) + completion_tokens = completion_response.get("usage", {}).get( + "completion_tokens", 0 + ) + total_time = completion_response.get("_response_ms", 0) + verbose_logger.debug( + f"completion_response response ms: {completion_response.get('_response_ms')} " + ) + model = model or completion_response.get( + "model", None + ) # check if user passed an override for model, if it's none check completion_response['model'] + if hasattr(completion_response, "_hidden_params"): + if ( + completion_response._hidden_params.get("model", None) is not None + and len(completion_response._hidden_params["model"]) > 0 + ): + model = completion_response._hidden_params.get("model", model) + custom_llm_provider = completion_response._hidden_params.get( + "custom_llm_provider", "" + ) + region_name = completion_response._hidden_params.get( + "region_name", region_name + ) + size = completion_response._hidden_params.get( + "optional_params", {} + ).get( + "size", "1024-x-1024" + ) # openai default + quality = completion_response._hidden_params.get( + "optional_params", {} + ).get( + "quality", "standard" + ) # openai default + n = completion_response._hidden_params.get("optional_params", {}).get( + "n", 1 + ) # openai default + else: + if len(messages) > 0: + prompt_tokens = token_counter(model=model, messages=messages) + elif len(prompt) > 0: + prompt_tokens = token_counter(model=model, text=prompt) + completion_tokens = token_counter(model=model, text=completion) + if model is None: + raise ValueError( + f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" + ) + + if ( + call_type == CallTypes.image_generation.value + or call_type == CallTypes.aimage_generation.value + ): + ### IMAGE GENERATION COST CALCULATION ### + if custom_llm_provider == "vertex_ai": + # https://cloud.google.com/vertex-ai/generative-ai/pricing + # Vertex Charges Flat $0.20 per image + return 0.020 + + # fix size to match naming convention + if "x" in size and "-x-" not in size: + size = size.replace("x", "-x-") + image_gen_model_name = f"{size}/{model}" + image_gen_model_name_with_quality = image_gen_model_name + if quality is not None: + image_gen_model_name_with_quality = f"{quality}/{image_gen_model_name}" + size = size.split("-x-") + height = int(size[0]) # if it's 1024-x-1024 vs. 1024x1024 + width = int(size[1]) + verbose_logger.debug(f"image_gen_model_name: {image_gen_model_name}") + verbose_logger.debug( + f"image_gen_model_name_with_quality: {image_gen_model_name_with_quality}" + ) + if image_gen_model_name in litellm.model_cost: + return ( + litellm.model_cost[image_gen_model_name]["input_cost_per_pixel"] + * height + * width + * n + ) + elif image_gen_model_name_with_quality in litellm.model_cost: + return ( + litellm.model_cost[image_gen_model_name_with_quality][ + "input_cost_per_pixel" + ] + * height + * width + * n + ) + else: + raise Exception( + f"Model={image_gen_model_name} not found in completion cost model map" + ) + # Calculate cost based on prompt_tokens, completion_tokens + if ( + "togethercomputer" in model + or "together_ai" in model + or custom_llm_provider == "together_ai" + ): + # together ai prices based on size of llm + # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json + model = get_model_params_and_category(model) + # replicate llms are calculate based on time for request running + # see https://replicate.com/pricing + elif ( + model in litellm.replicate_models or "replicate" in model + ) and model not in litellm.model_cost: + # for unmapped replicate model, default to replicate's time tracking logic + return get_replicate_completion_pricing(completion_response, total_time) + + if model is None: + raise ValueError( + f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" + ) + + ( + prompt_tokens_cost_usd_dollar, + completion_tokens_cost_usd_dollar, + ) = cost_per_token( + model=model, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + custom_llm_provider=custom_llm_provider, + response_time_ms=total_time, + region_name=region_name, + custom_cost_per_second=custom_cost_per_second, + custom_cost_per_token=custom_cost_per_token, + ) + _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar + print_verbose( + f"final cost: {_final_cost}; prompt_tokens_cost_usd_dollar: {prompt_tokens_cost_usd_dollar}; completion_tokens_cost_usd_dollar: {completion_tokens_cost_usd_dollar}" + ) + return _final_cost + except Exception as e: + raise e + + +def response_cost_calculator( + response_object: Union[ + ModelResponse, + EmbeddingResponse, + ImageResponse, + TranscriptionResponse, + TextCompletionResponse, + ], + model: str, + custom_llm_provider: str, + call_type: Literal[ + "embedding", + "aembedding", + "completion", + "acompletion", + "atext_completion", + "text_completion", + "image_generation", + "aimage_generation", + "moderation", + "amoderation", + "atranscription", + "transcription", + "aspeech", + "speech", + ], + optional_params: dict, + cache_hit: Optional[bool] = None, + base_model: Optional[str] = None, + custom_pricing: Optional[bool] = None, +) -> Optional[float]: + try: + response_cost: float = 0.0 + if cache_hit is not None and cache_hit is True: + response_cost = 0.0 + else: + response_object._hidden_params["optional_params"] = optional_params + if isinstance(response_object, ImageResponse): + response_cost = completion_cost( + completion_response=response_object, + model=model, + call_type=call_type, + custom_llm_provider=custom_llm_provider, + ) + else: + if ( + model in litellm.model_cost + and custom_pricing is not None + and custom_llm_provider is True + ): # override defaults if custom pricing is set + base_model = model + # base_model defaults to None if not set on model_info + response_cost = completion_cost( + completion_response=response_object, + call_type=call_type, + model=base_model, + custom_llm_provider=custom_llm_provider, + ) + return response_cost + except litellm.NotFoundError as e: + print_verbose( + f"Model={model} for LLM Provider={custom_llm_provider} not found in completion cost map." + ) + return None diff --git a/litellm/exceptions.py b/litellm/exceptions.py index d189b7ebe..edc17133d 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -20,18 +20,44 @@ class AuthenticationError(openai.AuthenticationError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 401 self.message = message self.llm_provider = llm_provider self.model = model self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries + self.response = response or httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="GET", url="https://litellm.ai" + ), # mock request object + ) super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + # raise when invalid models passed, example gpt-8 class NotFoundError(openai.NotFoundError): # type: ignore @@ -40,18 +66,44 @@ class NotFoundError(openai.NotFoundError): # type: ignore message, model, llm_provider, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 404 self.message = message self.model = model self.llm_provider = llm_provider self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries + self.response = response or httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="GET", url="https://litellm.ai" + ), # mock request object + ) super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class BadRequestError(openai.BadRequestError): # type: ignore def __init__( @@ -61,6 +113,8 @@ class BadRequestError(openai.BadRequestError): # type: ignore llm_provider, response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 400 self.message = message @@ -73,10 +127,28 @@ class BadRequestError(openai.BadRequestError): # type: ignore method="GET", url="https://litellm.ai" ), # mock request object ) + self.max_retries = max_retries + self.num_retries = num_retries super().__init__( self.message, response=response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore def __init__( @@ -86,20 +158,46 @@ class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore llm_provider, response: httpx.Response, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 422 self.message = message self.model = model self.llm_provider = llm_provider self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries super().__init__( self.message, response=response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class Timeout(openai.APITimeoutError): # type: ignore def __init__( - self, message, model, llm_provider, litellm_debug_info: Optional[str] = None + self, + message, + model, + llm_provider, + litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): request = httpx.Request(method="POST", url="https://api.openai.com/v1") super().__init__( @@ -110,10 +208,25 @@ class Timeout(openai.APITimeoutError): # type: ignore self.model = model self.llm_provider = llm_provider self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries # custom function to convert to str def __str__(self): - return str(self.message) + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message class PermissionDeniedError(openai.PermissionDeniedError): # type:ignore @@ -124,16 +237,36 @@ class PermissionDeniedError(openai.PermissionDeniedError): # type:ignore model, response: httpx.Response, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 403 self.message = message self.llm_provider = llm_provider self.model = model self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries super().__init__( self.message, response=response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class RateLimitError(openai.RateLimitError): # type: ignore def __init__( @@ -141,18 +274,48 @@ class RateLimitError(openai.RateLimitError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 429 self.message = message self.llm_provider = llm_provider - self.modle = model + self.model = model self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries + if response is None: + self.response = httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ) + else: + self.response = response super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + # sub class of rate limit error - meant to give more granularity for error handling context window exceeded errors class ContextWindowExceededError(BadRequestError): # type: ignore @@ -174,8 +337,25 @@ class ContextWindowExceededError(BadRequestError): # type: ignore model=self.model, # type: ignore llm_provider=self.llm_provider, # type: ignore response=response, + litellm_debug_info=self.litellm_debug_info, ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + # sub class of bad request error - meant to help us catch guardrails-related errors on proxy. class RejectedRequestError(BadRequestError): # type: ignore @@ -200,8 +380,25 @@ class RejectedRequestError(BadRequestError): # type: ignore model=self.model, # type: ignore llm_provider=self.llm_provider, # type: ignore response=response, + litellm_debug_info=self.litellm_debug_info, ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class ContentPolicyViolationError(BadRequestError): # type: ignore # Error code: 400 - {'error': {'code': 'content_policy_violation', 'message': 'Your request was rejected as a result of our safety system. Image descriptions generated from your prompt may contain text that is not allowed by our safety system. If you believe this was done in error, your request may succeed if retried, or by adjusting your prompt.', 'param': None, 'type': 'invalid_request_error'}} @@ -223,8 +420,25 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore model=self.model, # type: ignore llm_provider=self.llm_provider, # type: ignore response=response, + litellm_debug_info=self.litellm_debug_info, ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class ServiceUnavailableError(openai.APIStatusError): # type: ignore def __init__( @@ -232,18 +446,97 @@ class ServiceUnavailableError(openai.APIStatusError): # type: ignore message, llm_provider, model, - response: httpx.Response, + response: Optional[httpx.Response] = None, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = 503 self.message = message self.llm_provider = llm_provider self.model = model self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries + if response is None: + self.response = httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ) + else: + self.response = response super().__init__( - self.message, response=response, body=None + self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + +class InternalServerError(openai.InternalServerError): # type: ignore + def __init__( + self, + message, + llm_provider, + model, + response: Optional[httpx.Response] = None, + litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, + ): + self.status_code = 500 + self.message = message + self.llm_provider = llm_provider + self.model = model + self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries + if response is None: + self.response = httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ) + else: + self.response = response + super().__init__( + self.message, response=self.response, body=None + ) # Call the base class constructor with the parameters it needs + + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + # raise this when the API returns an invalid response object - https://github.com/openai/openai-python/blob/1be14ee34a0f8e42d3f9aa5451aa4cb161f1781f/openai/api_requestor.py#L401 class APIError(openai.APIError): # type: ignore @@ -255,14 +548,34 @@ class APIError(openai.APIError): # type: ignore model, request: httpx.Request, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.status_code = status_code self.message = message self.llm_provider = llm_provider self.model = model self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries super().__init__(self.message, request=request, body=None) # type: ignore + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + # raised if an invalid request (not get, delete, put, post) is made class APIConnectionError(openai.APIConnectionError): # type: ignore @@ -273,19 +586,45 @@ class APIConnectionError(openai.APIConnectionError): # type: ignore model, request: httpx.Request, litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.message = message self.llm_provider = llm_provider self.model = model self.status_code = 500 self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries super().__init__(message=self.message, request=request) + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + # raised if an invalid request (not get, delete, put, post) is made class APIResponseValidationError(openai.APIResponseValidationError): # type: ignore def __init__( - self, message, llm_provider, model, litellm_debug_info: Optional[str] = None + self, + message, + llm_provider, + model, + litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, ): self.message = message self.llm_provider = llm_provider @@ -293,8 +632,26 @@ class APIResponseValidationError(openai.APIResponseValidationError): # type: ig request = httpx.Request(method="POST", url="https://api.openai.com/v1") response = httpx.Response(status_code=500, request=request) self.litellm_debug_info = litellm_debug_info + self.max_retries = max_retries + self.num_retries = num_retries super().__init__(response=response, body=None, message=message) + def __str__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + + def __repr__(self): + _message = self.message + if self.num_retries: + _message += f" LiteLLM Retried: {self.num_retries} times" + if self.max_retries: + _message += f", LiteLLM Max Retries: {self.max_retries}" + return _message + class OpenAIError(openai.OpenAIError): # type: ignore def __init__(self, original_exception): @@ -309,11 +666,33 @@ class OpenAIError(openai.OpenAIError): # type: ignore self.llm_provider = "openai" +LITELLM_EXCEPTION_TYPES = [ + AuthenticationError, + NotFoundError, + BadRequestError, + UnprocessableEntityError, + Timeout, + PermissionDeniedError, + RateLimitError, + ContextWindowExceededError, + RejectedRequestError, + ContentPolicyViolationError, + InternalServerError, + ServiceUnavailableError, + APIError, + APIConnectionError, + APIResponseValidationError, + OpenAIError, + InternalServerError, +] + + class BudgetExceededError(Exception): def __init__(self, current_cost, max_budget): self.current_cost = current_cost self.max_budget = max_budget message = f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}" + self.message = message super().__init__(message) diff --git a/litellm/integrations/aispend.py b/litellm/integrations/aispend.py index 2fe8ea0df..ca284e62e 100644 --- a/litellm/integrations/aispend.py +++ b/litellm/integrations/aispend.py @@ -169,6 +169,5 @@ class AISpendLogger: print_verbose(f"AISpend Logging - final data object: {data}") except: - # traceback.print_exc() print_verbose(f"AISpend Logging Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/berrispend.py b/litellm/integrations/berrispend.py index 7d30b706c..d428fb54d 100644 --- a/litellm/integrations/berrispend.py +++ b/litellm/integrations/berrispend.py @@ -178,6 +178,5 @@ class BerriSpendLogger: print_verbose(f"BerriSpend Logging - final data object: {data}") response = requests.post(url, headers=headers, json=data) except: - # traceback.print_exc() print_verbose(f"BerriSpend Logging Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 0c38b8626..f8b6b1bbf 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -297,6 +297,5 @@ class ClickhouseLogger: # make request to endpoint with payload verbose_logger.debug(f"Clickhouse Logger - final response = {response}") except Exception as e: - traceback.print_exc() verbose_logger.debug(f"Clickhouse - {str(e)}\n{traceback.format_exc()}") pass diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index e192cdaea..1d447da1f 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -115,7 +115,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac ) print_verbose(f"Custom Logger - model call details: {kwargs}") except: - traceback.print_exc() print_verbose(f"Custom Logger Error - {traceback.format_exc()}") async def async_log_input_event( @@ -130,7 +129,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac ) print_verbose(f"Custom Logger - model call details: {kwargs}") except: - traceback.print_exc() print_verbose(f"Custom Logger Error - {traceback.format_exc()}") def log_event( @@ -146,7 +144,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac end_time, ) except: - # traceback.print_exc() print_verbose(f"Custom Logger Error - {traceback.format_exc()}") pass @@ -163,6 +160,5 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac end_time, ) except: - # traceback.print_exc() print_verbose(f"Custom Logger Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/datadog.py b/litellm/integrations/datadog.py index 6d5e08faf..d835b3d67 100644 --- a/litellm/integrations/datadog.py +++ b/litellm/integrations/datadog.py @@ -134,7 +134,6 @@ class DataDogLogger: f"Datadog Layer Logging - final response object: {response_obj}" ) except Exception as e: - traceback.print_exc() verbose_logger.debug( f"Datadog Layer Error - {str(e)}\n{traceback.format_exc()}" ) diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 21ccabe4b..847f930ec 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -85,6 +85,5 @@ class DyanmoDBLogger: ) return response except: - traceback.print_exc() print_verbose(f"DynamoDB Layer Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/email_templates/templates.py b/litellm/integrations/email_templates/templates.py new file mode 100644 index 000000000..7029e8ce1 --- /dev/null +++ b/litellm/integrations/email_templates/templates.py @@ -0,0 +1,62 @@ +""" +Email Templates used by the LiteLLM Email Service in slack_alerting.py +""" + +KEY_CREATED_EMAIL_TEMPLATE = """ + LiteLLM Logo + +

Hi {recipient_email},
+ + I'm happy to provide you with an OpenAI Proxy API Key, loaded with ${key_budget} per month.

+ + + Key:

{key_token}

+
+ +

Usage Example

+ + Detailed Documentation on Usage with OpenAI Python SDK, Langchain, LlamaIndex, Curl + +
+
+                    import openai
+                    client = openai.OpenAI(
+                        api_key="{key_token}",
+                        base_url={{base_url}}
+                    )
+
+                    response = client.chat.completions.create(
+                        model="gpt-3.5-turbo", # model to send to the proxy
+                        messages = [
+                            {{
+                                "role": "user",
+                                "content": "this is a test request, write a short poem"
+                            }}
+                        ]
+                    )
+
+                    
+ + + If you have any questions, please send an email to {email_support_contact}

+ + Best,
+ The LiteLLM team
+""" + + +USER_INVITED_EMAIL_TEMPLATE = """ + LiteLLM Logo + +

Hi {recipient_email},
+ + You were invited to use OpenAI Proxy API for team {team_name}

+ + Get Started here

+ + + If you have any questions, please send an email to {email_support_contact}

+ + Best,
+ The LiteLLM team
+""" diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index 85e73258e..8ea18a7d5 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -112,6 +112,5 @@ class HeliconeLogger: ) print_verbose(f"Helicone Logging - Error {response.text}") except: - # traceback.print_exc() print_verbose(f"Helicone Logging Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index a9c67547d..7fe2e9f22 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -69,6 +69,43 @@ class LangFuseLogger: else: self.upstream_langfuse = None + @staticmethod + def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: + """ + Adds metadata from proxy request headers to Langfuse logging if keys start with "langfuse_" + and overwrites litellm_params.metadata if already included. + + For example if you want to append your trace to an existing `trace_id` via header, send + `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request. + """ + if litellm_params is None: + return metadata + + if litellm_params.get("proxy_server_request") is None: + return metadata + + if metadata is None: + metadata = {} + + proxy_headers = ( + litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} + ) + + for metadata_param_key in proxy_headers: + if metadata_param_key.startswith("langfuse_"): + trace_param_key = metadata_param_key.replace("langfuse_", "", 1) + if trace_param_key in metadata: + verbose_logger.warning( + f"Overwriting Langfuse `{trace_param_key}` from request header" + ) + else: + verbose_logger.debug( + f"Found Langfuse `{trace_param_key}` in request header" + ) + metadata[trace_param_key] = proxy_headers.get(metadata_param_key) + + return metadata + # def log_error(kwargs, response_obj, start_time, end_time): # generation = trace.generation( # level ="ERROR" # can be any of DEBUG, DEFAULT, WARNING or ERROR @@ -97,6 +134,7 @@ class LangFuseLogger: metadata = ( litellm_params.get("metadata", {}) or {} ) # if litellm_params['metadata'] == None + metadata = self.add_metadata_from_header(litellm_params, metadata) optional_params = copy.deepcopy(kwargs.get("optional_params", {})) prompt = {"messages": kwargs.get("messages")} @@ -182,9 +220,11 @@ class LangFuseLogger: verbose_logger.info(f"Langfuse Layer Logging - logging success") return {"trace_id": trace_id, "generation_id": generation_id} - except: - traceback.print_exc() - verbose_logger.debug(f"Langfuse Layer Error - {traceback.format_exc()}") + except Exception as e: + verbose_logger.error( + "Langfuse Layer Error(): Exception occured - {}".format(str(e)) + ) + verbose_logger.debug(traceback.format_exc()) return {"trace_id": None, "generation_id": None} async def _async_log_event( @@ -396,6 +436,8 @@ class LangFuseLogger: cost = kwargs.get("response_cost", None) print_verbose(f"trace: {cost}") + clean_metadata["litellm_response_cost"] = cost + if ( litellm._langfuse_default_tags is not None and isinstance(litellm._langfuse_default_tags, list) @@ -455,8 +497,13 @@ class LangFuseLogger: } generation_name = clean_metadata.pop("generation_name", None) if generation_name is None: - # just log `litellm-{call_type}` as the generation name + # if `generation_name` is None, use sensible default values + # If using litellm proxy user `key_alias` if not None + # If `key_alias` is None, just log `litellm-{call_type}` as the generation name + _user_api_key_alias = clean_metadata.get("user_api_key_alias", None) generation_name = f"litellm-{kwargs.get('call_type', 'completion')}" + if _user_api_key_alias is not None: + generation_name = f"litellm:{_user_api_key_alias}" if response_obj is not None and "system_fingerprint" in response_obj: system_fingerprint = response_obj.get("system_fingerprint", None) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 3e25b4ee7..48185afee 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -44,7 +44,9 @@ class LangsmithLogger: print_verbose( f"Langsmith Logging - project_name: {project_name}, run_name {run_name}" ) - langsmith_base_url = os.getenv("LANGSMITH_BASE_URL", "https://api.smith.langchain.com") + langsmith_base_url = os.getenv( + "LANGSMITH_BASE_URL", "https://api.smith.langchain.com" + ) try: print_verbose( @@ -89,9 +91,7 @@ class LangsmithLogger: } url = f"{langsmith_base_url}/runs" - print_verbose( - f"Langsmith Logging - About to send data to {url} ..." - ) + print_verbose(f"Langsmith Logging - About to send data to {url} ...") response = requests.post( url=url, json=data, @@ -106,6 +106,5 @@ class LangsmithLogger: f"Langsmith Layer Logging - final response object: {response_obj}" ) except: - # traceback.print_exc() print_verbose(f"Langsmith Layer Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index e27d848fb..b4ab00820 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -171,7 +171,6 @@ class LogfireLogger: f"Logfire Layer Logging - final response object: {response_obj}" ) except Exception as e: - traceback.print_exc() verbose_logger.debug( f"Logfire Layer Error - {str(e)}\n{traceback.format_exc()}" ) diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 2e16e44a1..141ea6488 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -14,6 +14,7 @@ def parse_usage(usage): "prompt": usage["prompt_tokens"] if "prompt_tokens" in usage else 0, } + def parse_tool_calls(tool_calls): if tool_calls is None: return None @@ -26,13 +27,13 @@ def parse_tool_calls(tool_calls): "function": { "name": tool_call.function.name, "arguments": tool_call.function.arguments, - } + }, } return serialized - + return [clean_tool_call(tool_call) for tool_call in tool_calls] - + def parse_messages(input): @@ -176,6 +177,5 @@ class LunaryLogger: ) except: - # traceback.print_exc() print_verbose(f"Lunary Logging Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py new file mode 100644 index 000000000..bb9e34b1a --- /dev/null +++ b/litellm/integrations/opentelemetry.py @@ -0,0 +1,547 @@ +import os +from dataclasses import dataclass +from datetime import datetime +import litellm + +from litellm.integrations.custom_logger import CustomLogger +from litellm._logging import verbose_logger +from litellm.types.services import ServiceLoggerPayload +from typing import Union, Optional, TYPE_CHECKING, Any + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth + + Span = _Span + UserAPIKeyAuth = _UserAPIKeyAuth +else: + Span = Any + UserAPIKeyAuth = Any + + +LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") +LITELLM_RESOURCE = { + "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), +} +RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" +LITELLM_REQUEST_SPAN_NAME = "litellm_request" + + +@dataclass +class OpenTelemetryConfig: + from opentelemetry.sdk.trace.export import SpanExporter + + exporter: str | SpanExporter = "console" + endpoint: Optional[str] = None + headers: Optional[str] = None + + @classmethod + def from_env(cls): + """ + OTEL_HEADERS=x-honeycomb-team=B85YgLm9**** + OTEL_EXPORTER="otlp_http" + OTEL_ENDPOINT="https://api.honeycomb.io/v1/traces" + + OTEL_HEADERS gets sent as headers = {"x-honeycomb-team": "B85YgLm96******"} + """ + return cls( + exporter=os.getenv("OTEL_EXPORTER", "console"), + endpoint=os.getenv("OTEL_ENDPOINT"), + headers=os.getenv( + "OTEL_HEADERS" + ), # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96VGdFisfJVme1H" + ) + + +class OpenTelemetry(CustomLogger): + def __init__(self, config=OpenTelemetryConfig.from_env()): + from opentelemetry import trace + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + + self.config = config + self.OTEL_EXPORTER = self.config.exporter + self.OTEL_ENDPOINT = self.config.endpoint + self.OTEL_HEADERS = self.config.headers + provider = TracerProvider(resource=Resource(attributes=LITELLM_RESOURCE)) + provider.add_span_processor(self._get_span_processor()) + + trace.set_tracer_provider(provider) + self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) + + _debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower() + + if _debug_otel == "true": + # Set up logging + import logging + + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger(__name__) + + # Enable OpenTelemetry logging + otel_exporter_logger = logging.getLogger("opentelemetry.sdk.trace.export") + otel_exporter_logger.setLevel(logging.DEBUG) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self._handle_sucess(kwargs, response_obj, start_time, end_time) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._handle_failure(kwargs, response_obj, start_time, end_time) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self._handle_sucess(kwargs, response_obj, start_time, end_time) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._handle_failure(kwargs, response_obj, start_time, end_time) + + async def async_service_success_hook( + self, + payload: ServiceLoggerPayload, + parent_otel_span: Optional[Span] = None, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + ): + from opentelemetry import trace + from datetime import datetime + from opentelemetry.trace import Status, StatusCode + + _start_time_ns = start_time + _end_time_ns = end_time + + if isinstance(start_time, float): + _start_time_ns = int(int(start_time) * 1e9) + else: + _start_time_ns = self._to_ns(start_time) + + if isinstance(end_time, float): + _end_time_ns = int(int(end_time) * 1e9) + else: + _end_time_ns = self._to_ns(end_time) + + if parent_otel_span is not None: + _span_name = payload.service + service_logging_span = self.tracer.start_span( + name=_span_name, + context=trace.set_span_in_context(parent_otel_span), + start_time=_start_time_ns, + ) + service_logging_span.set_attribute(key="call_type", value=payload.call_type) + service_logging_span.set_attribute( + key="service", value=payload.service.value + ) + service_logging_span.set_status(Status(StatusCode.OK)) + service_logging_span.end(end_time=_end_time_ns) + + async def async_service_failure_hook( + self, + payload: ServiceLoggerPayload, + parent_otel_span: Optional[Span] = None, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + ): + from opentelemetry import trace + from datetime import datetime + from opentelemetry.trace import Status, StatusCode + + _start_time_ns = start_time + _end_time_ns = end_time + + if isinstance(start_time, float): + _start_time_ns = int(int(start_time) * 1e9) + else: + _start_time_ns = self._to_ns(start_time) + + if isinstance(end_time, float): + _end_time_ns = int(int(end_time) * 1e9) + else: + _end_time_ns = self._to_ns(end_time) + + if parent_otel_span is not None: + _span_name = payload.service + service_logging_span = self.tracer.start_span( + name=_span_name, + context=trace.set_span_in_context(parent_otel_span), + start_time=_start_time_ns, + ) + service_logging_span.set_attribute(key="call_type", value=payload.call_type) + service_logging_span.set_attribute( + key="service", value=payload.service.value + ) + service_logging_span.set_status(Status(StatusCode.ERROR)) + service_logging_span.end(end_time=_end_time_ns) + + async def async_post_call_failure_hook( + self, original_exception: Exception, user_api_key_dict: UserAPIKeyAuth + ): + from opentelemetry.trace import Status, StatusCode + from opentelemetry import trace + + parent_otel_span = user_api_key_dict.parent_otel_span + if parent_otel_span is not None: + parent_otel_span.set_status(Status(StatusCode.ERROR)) + _span_name = "Failed Proxy Server Request" + + # Exception Logging Child Span + exception_logging_span = self.tracer.start_span( + name=_span_name, + context=trace.set_span_in_context(parent_otel_span), + ) + exception_logging_span.set_attribute( + key="exception", value=str(original_exception) + ) + exception_logging_span.set_status(Status(StatusCode.ERROR)) + exception_logging_span.end(end_time=self._to_ns(datetime.now())) + + # End Parent OTEL Sspan + parent_otel_span.end(end_time=self._to_ns(datetime.now())) + + def _handle_sucess(self, kwargs, response_obj, start_time, end_time): + from opentelemetry.trace import Status, StatusCode + from opentelemetry import trace + + verbose_logger.debug( + "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s", + kwargs, + self.config, + ) + _parent_context, parent_otel_span = self._get_span_context(kwargs) + + # Span 1: Requst sent to litellm SDK + span = self.tracer.start_span( + name=self._get_span_name(kwargs), + start_time=self._to_ns(start_time), + context=_parent_context, + ) + span.set_status(Status(StatusCode.OK)) + self.set_attributes(span, kwargs, response_obj) + + if litellm.turn_off_message_logging is True: + pass + else: + # Span 2: Raw Request / Response to LLM + raw_request_span = self.tracer.start_span( + name=RAW_REQUEST_SPAN_NAME, + start_time=self._to_ns(start_time), + context=trace.set_span_in_context(span), + ) + + raw_request_span.set_status(Status(StatusCode.OK)) + self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) + raw_request_span.end(end_time=self._to_ns(end_time)) + + span.end(end_time=self._to_ns(end_time)) + + if parent_otel_span is not None: + parent_otel_span.end(end_time=self._to_ns(datetime.now())) + + def _handle_failure(self, kwargs, response_obj, start_time, end_time): + from opentelemetry.trace import Status, StatusCode + + span = self.tracer.start_span( + name=self._get_span_name(kwargs), + start_time=self._to_ns(start_time), + context=self._get_span_context(kwargs), + ) + span.set_status(Status(StatusCode.ERROR)) + self.set_attributes(span, kwargs, response_obj) + span.end(end_time=self._to_ns(end_time)) + + def set_tools_attributes(self, span: Span, tools): + from opentelemetry.semconv.ai import SpanAttributes + import json + + if not tools: + return + + try: + for i, tool in enumerate(tools): + function = tool.get("function") + if not function: + continue + + prefix = f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}" + span.set_attribute(f"{prefix}.name", function.get("name")) + span.set_attribute(f"{prefix}.description", function.get("description")) + span.set_attribute( + f"{prefix}.parameters", json.dumps(function.get("parameters")) + ) + except Exception as e: + verbose_logger.error( + "OpenTelemetry: Error setting tools attributes: %s", str(e) + ) + pass + + def set_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + optional_params = kwargs.get("optional_params", {}) + litellm_params = kwargs.get("litellm_params", {}) or {} + + # https://github.com/open-telemetry/semantic-conventions/blob/main/model/registry/gen-ai.yaml + # Following Conventions here: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/llm-spans.md + + ############################################# + ########## LLM Request Attributes ########### + ############################################# + + # The name of the LLM a request is being made to + if kwargs.get("model"): + span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")) + + # The Generative AI Provider: Azure, OpenAI, etc. + span.set_attribute( + SpanAttributes.LLM_SYSTEM, + litellm_params.get("custom_llm_provider", "Unknown"), + ) + + # The maximum number of tokens the LLM generates for a request. + if optional_params.get("max_tokens"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MAX_TOKENS, optional_params.get("max_tokens") + ) + + # The temperature setting for the LLM request. + if optional_params.get("temperature"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TEMPERATURE, + optional_params.get("temperature"), + ) + + # The top_p sampling setting for the LLM request. + if optional_params.get("top_p"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TOP_P, optional_params.get("top_p") + ) + + span.set_attribute( + SpanAttributes.LLM_IS_STREAMING, optional_params.get("stream", False) + ) + + if optional_params.get("tools"): + tools = optional_params["tools"] + self.set_tools_attributes(span, tools) + + if optional_params.get("user"): + span.set_attribute(SpanAttributes.LLM_USER, optional_params.get("user")) + + if kwargs.get("messages"): + for idx, prompt in enumerate(kwargs.get("messages")): + if prompt.get("role"): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", + prompt.get("role"), + ) + + if prompt.get("content"): + if not isinstance(prompt.get("content"), str): + prompt["content"] = str(prompt.get("content")) + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", + prompt.get("content"), + ) + ############################################# + ########## LLM Response Attributes ########## + ############################################# + if response_obj.get("choices"): + for idx, choice in enumerate(response_obj.get("choices")): + if choice.get("finish_reason"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", + choice.get("finish_reason"), + ) + if choice.get("message"): + if choice.get("message").get("role"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", + choice.get("message").get("role"), + ) + if choice.get("message").get("content"): + if not isinstance(choice.get("message").get("content"), str): + choice["message"]["content"] = str( + choice.get("message").get("content") + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", + choice.get("message").get("content"), + ) + + message = choice.get("message") + tool_calls = message.get("tool_calls") + if tool_calls: + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.function_call.name", + tool_calls[0].get("function").get("name"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.function_call.arguments", + tool_calls[0].get("function").get("arguments"), + ) + + # The unique identifier for the completion. + if response_obj.get("id"): + span.set_attribute("gen_ai.response.id", response_obj.get("id")) + + # The model used to generate the response. + if response_obj.get("model"): + span.set_attribute( + SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model") + ) + + usage = response_obj.get("usage") + if usage: + span.set_attribute( + SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + usage.get("total_tokens"), + ) + + # The number of tokens used in the LLM response (completion). + span.set_attribute( + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + usage.get("completion_tokens"), + ) + + # The number of tokens used in the LLM prompt. + span.set_attribute( + SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + usage.get("prompt_tokens"), + ) + + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + optional_params = kwargs.get("optional_params", {}) + litellm_params = kwargs.get("litellm_params", {}) or {} + custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") + + _raw_response = kwargs.get("original_response") + _additional_args = kwargs.get("additional_args", {}) or {} + complete_input_dict = _additional_args.get("complete_input_dict") + ############################################# + ########## LLM Request Attributes ########### + ############################################# + + # OTEL Attributes for the RAW Request to https://docs.anthropic.com/en/api/messages + if complete_input_dict: + for param, val in complete_input_dict.items(): + if not isinstance(val, str): + val = str(val) + span.set_attribute( + f"llm.{custom_llm_provider}.{param}", + val, + ) + + ############################################# + ########## LLM Response Attributes ########## + ############################################# + if _raw_response: + # cast sr -> dict + import json + + _raw_response = json.loads(_raw_response) + for param, val in _raw_response.items(): + if not isinstance(val, str): + val = str(val) + span.set_attribute( + f"llm.{custom_llm_provider}.{param}", + val, + ) + + pass + + def _to_ns(self, dt): + return int(dt.timestamp() * 1e9) + + def _get_span_name(self, kwargs): + return LITELLM_REQUEST_SPAN_NAME + + def _get_span_context(self, kwargs): + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + from opentelemetry import trace + + litellm_params = kwargs.get("litellm_params", {}) or {} + proxy_server_request = litellm_params.get("proxy_server_request", {}) or {} + headers = proxy_server_request.get("headers", {}) or {} + traceparent = headers.get("traceparent", None) + _metadata = litellm_params.get("metadata", {}) or {} + parent_otel_span = _metadata.get("litellm_parent_otel_span", None) + + """ + Two way to use parents in opentelemetry + - using the traceparent header + - using the parent_otel_span in the [metadata][parent_otel_span] + """ + if parent_otel_span is not None: + return trace.set_span_in_context(parent_otel_span), parent_otel_span + + if traceparent is None: + return None, None + else: + carrier = {"traceparent": traceparent} + return TraceContextTextMapPropagator().extract(carrier=carrier), None + + def _get_span_processor(self): + from opentelemetry.sdk.trace.export import ( + SpanExporter, + SimpleSpanProcessor, + BatchSpanProcessor, + ConsoleSpanExporter, + ) + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as OTLPSpanExporterHTTP, + ) + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as OTLPSpanExporterGRPC, + ) + + verbose_logger.debug( + "OpenTelemetry Logger, initializing span processor \nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s", + self.OTEL_EXPORTER, + self.OTEL_ENDPOINT, + self.OTEL_HEADERS, + ) + _split_otel_headers = {} + if self.OTEL_HEADERS is not None and isinstance(self.OTEL_HEADERS, str): + _split_otel_headers = self.OTEL_HEADERS.split("=") + _split_otel_headers = {_split_otel_headers[0]: _split_otel_headers[1]} + + if isinstance(self.OTEL_EXPORTER, SpanExporter): + verbose_logger.debug( + "OpenTelemetry: intiializing SpanExporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return SimpleSpanProcessor(self.OTEL_EXPORTER) + + if self.OTEL_EXPORTER == "console": + verbose_logger.debug( + "OpenTelemetry: intiializing console exporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return BatchSpanProcessor(ConsoleSpanExporter()) + elif self.OTEL_EXPORTER == "otlp_http": + verbose_logger.debug( + "OpenTelemetry: intiializing http exporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return BatchSpanProcessor( + OTLPSpanExporterHTTP( + endpoint=self.OTEL_ENDPOINT, headers=_split_otel_headers + ) + ) + elif self.OTEL_EXPORTER == "otlp_grpc": + verbose_logger.debug( + "OpenTelemetry: intiializing grpc exporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return BatchSpanProcessor( + OTLPSpanExporterGRPC( + endpoint=self.OTEL_ENDPOINT, headers=_split_otel_headers + ) + ) + else: + verbose_logger.debug( + "OpenTelemetry: intiializing console exporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return BatchSpanProcessor(ConsoleSpanExporter()) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 6fbc6ca4c..af0d1d310 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -109,8 +109,8 @@ class PrometheusLogger: end_user_id, user_api_key, model, user_api_team, user_id ).inc() except Exception as e: - traceback.print_exc() - verbose_logger.debug( - f"prometheus Layer Error - {str(e)}\n{traceback.format_exc()}" + verbose_logger.error( + "prometheus Layer Error(): Exception occured - {}".format(str(e)) ) + verbose_logger.debug(traceback.format_exc()) pass diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index d131e44f0..0796d1048 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -180,6 +180,5 @@ class S3Logger: print_verbose(f"s3 Layer Logging - final response object: {response_obj}") return response except Exception as e: - traceback.print_exc() verbose_logger.debug(f"s3 Layer Error - {str(e)}\n{traceback.format_exc()}") pass diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 9e35b4fc3..21415fb6d 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -18,6 +18,7 @@ from litellm.proxy._types import WebhookEvent import random from typing import TypedDict from openai import APIError +from .email_templates.templates import * import litellm.types from litellm.types.router import LiteLLM_Params @@ -41,10 +42,7 @@ class ProviderRegionOutageModel(BaseOutageModel): # we use this for the email header, please send a test email if you change this. verify it looks good on email LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" -EMAIL_LOGO_URL = os.getenv( - "SMTP_SENDER_LOGO", "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" -) -EMAIL_SUPPORT_CONTACT = os.getenv("EMAIL_SUPPORT_CONTACT", "support@berri.ai") +LITELLM_SUPPORT_CONTACT = "support@berri.ai" class LiteLLMBase(BaseModel): @@ -328,8 +326,8 @@ class SlackAlerting(CustomLogger): end_time=end_time, ) ) - if litellm.turn_off_message_logging: - messages = "Message not logged. `litellm.turn_off_message_logging=True`." + if litellm.turn_off_message_logging or litellm.redact_messages_in_exceptions: + messages = "Message not logged. litellm.redact_messages_in_exceptions=True" request_info = f"\nRequest Model: `{model}`\nAPI Base: `{api_base}`\nMessages: `{messages}`" slow_message = f"`Responses are slow - {round(time_difference_float,2)}s response time > Alerting threshold: {self.alerting_threshold}s`" if time_difference_float > self.alerting_threshold: @@ -539,7 +537,7 @@ class SlackAlerting(CustomLogger): cache_list=combined_metrics_cache_keys ) - message += f"\n\nNext Run is in: `{time.time() + self.alerting_args.daily_report_frequency}`s" + message += f"\n\nNext Run is at: `{time.time() + self.alerting_args.daily_report_frequency}`s" # send alert await self.send_alert(message=message, level="Low", alert_type="daily_reports") @@ -569,9 +567,12 @@ class SlackAlerting(CustomLogger): except: messages = "" - if litellm.turn_off_message_logging: + if ( + litellm.turn_off_message_logging + or litellm.redact_messages_in_exceptions + ): messages = ( - "Message not logged. `litellm.turn_off_message_logging=True`." + "Message not logged. litellm.redact_messages_in_exceptions=True" ) request_info = f"\nRequest Model: `{model}`\nMessages: `{messages}`" else: @@ -687,14 +688,16 @@ class SlackAlerting(CustomLogger): event: Optional[ Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"] ] = None - event_group: Optional[Literal["user", "team", "key", "proxy"]] = None + event_group: Optional[ + Literal["internal_user", "team", "key", "proxy", "customer"] + ] = None event_message: str = "" webhook_event: Optional[WebhookEvent] = None if type == "proxy_budget": event_group = "proxy" event_message += "Proxy Budget: " elif type == "user_budget": - event_group = "user" + event_group = "internal_user" event_message += "User Budget: " _id = user_info.user_id or _id elif type == "team_budget": @@ -758,6 +761,36 @@ class SlackAlerting(CustomLogger): return return + async def customer_spend_alert( + self, + token: Optional[str], + key_alias: Optional[str], + end_user_id: Optional[str], + response_cost: Optional[float], + max_budget: Optional[float], + ): + if end_user_id is not None and token is not None and response_cost is not None: + # log customer spend + event = WebhookEvent( + spend=response_cost, + max_budget=max_budget, + token=token, + customer_id=end_user_id, + user_id=None, + team_id=None, + user_email=None, + key_alias=key_alias, + projected_exceeded_date=None, + projected_spend=None, + event="spend_tracked", + event_group="customer", + event_message="Customer spend tracked. Customer={}, spend={}".format( + end_user_id, response_cost + ), + ) + + await self.send_webhook_alert(webhook_event=event) + def _count_outage_alerts(self, alerts: List[int]) -> str: """ Parameters: @@ -1147,102 +1180,122 @@ Model Info: return False - async def send_key_created_email(self, webhook_event: WebhookEvent) -> bool: - from litellm.proxy.utils import send_email - - if self.alerting is None or "email" not in self.alerting: - # do nothing if user does not want email alerts - return False - - # make sure this is a premium user + async def _check_if_using_premium_email_feature( + self, + premium_user: bool, + email_logo_url: Optional[str] = None, + email_support_contact: Optional[str] = None, + ): from litellm.proxy.proxy_server import premium_user - from litellm.proxy.proxy_server import CommonProxyErrors, prisma_client + from litellm.proxy.proxy_server import CommonProxyErrors - if premium_user != True: - raise Exception( - f"Trying to use Email Alerting on key creation\n {CommonProxyErrors.not_premium_user.value}" + if premium_user is not True: + if email_logo_url is not None or email_support_contact is not None: + raise ValueError( + f"Trying to Customize Email Alerting\n {CommonProxyErrors.not_premium_user.value}" + ) + return + + async def send_key_created_or_user_invited_email( + self, webhook_event: WebhookEvent + ) -> bool: + try: + from litellm.proxy.utils import send_email + + if self.alerting is None or "email" not in self.alerting: + # do nothing if user does not want email alerts + return False + from litellm.proxy.proxy_server import premium_user, prisma_client + + email_logo_url = os.getenv( + "SMTP_SENDER_LOGO", os.getenv("EMAIL_LOGO_URL", None) + ) + email_support_contact = os.getenv("EMAIL_SUPPORT_CONTACT", None) + await self._check_if_using_premium_email_feature( + premium_user, email_logo_url, email_support_contact + ) + if email_logo_url is None: + email_logo_url = LITELLM_LOGO_URL + if email_support_contact is None: + email_support_contact = LITELLM_SUPPORT_CONTACT + + event_name = webhook_event.event_message + recipient_email = webhook_event.user_email + recipient_user_id = webhook_event.user_id + if ( + recipient_email is None + and recipient_user_id is not None + and prisma_client is not None + ): + user_row = await prisma_client.db.litellm_usertable.find_unique( + where={"user_id": recipient_user_id} + ) + + if user_row is not None: + recipient_email = user_row.user_email + + key_name = webhook_event.key_alias + key_token = webhook_event.token + key_budget = webhook_event.max_budget + base_url = os.getenv("PROXY_BASE_URL", "http://0.0.0.0:4000") + + email_html_content = "Alert from LiteLLM Server" + if recipient_email is None: + verbose_proxy_logger.error( + "Trying to send email alert to no recipient", + extra=webhook_event.dict(), + ) + + if webhook_event.event == "key_created": + email_html_content = KEY_CREATED_EMAIL_TEMPLATE.format( + email_logo_url=email_logo_url, + recipient_email=recipient_email, + key_budget=key_budget, + key_token=key_token, + base_url=base_url, + email_support_contact=email_support_contact, + ) + elif webhook_event.event == "internal_user_created": + # GET TEAM NAME + team_id = webhook_event.team_id + team_name = "Default Team" + if team_id is not None and prisma_client is not None: + team_row = await prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": team_id} + ) + if team_row is not None: + team_name = team_row.team_alias or "-" + email_html_content = USER_INVITED_EMAIL_TEMPLATE.format( + email_logo_url=email_logo_url, + recipient_email=recipient_email, + team_name=team_name, + base_url=base_url, + email_support_contact=email_support_contact, + ) + else: + verbose_proxy_logger.error( + "Trying to send email alert on unknown webhook event", + extra=webhook_event.model_dump(), + ) + + payload = webhook_event.model_dump_json() + email_event = { + "to": recipient_email, + "subject": f"LiteLLM: {event_name}", + "html": email_html_content, + } + + response = await send_email( + receiver_email=email_event["to"], + subject=email_event["subject"], + html=email_event["html"], ) - event_name = webhook_event.event_message - recipient_email = webhook_event.user_email - recipient_user_id = webhook_event.user_id - if ( - recipient_email is None - and recipient_user_id is not None - and prisma_client is not None - ): - user_row = await prisma_client.db.litellm_usertable.find_unique( - where={"user_id": recipient_user_id} - ) + return True - if user_row is not None: - recipient_email = user_row.user_email - - key_name = webhook_event.key_alias - key_token = webhook_event.token - key_budget = webhook_event.max_budget - - email_html_content = "Alert from LiteLLM Server" - if recipient_email is None: - verbose_proxy_logger.error( - "Trying to send email alert to no recipient", extra=webhook_event.dict() - ) - email_html_content = f""" - LiteLLM Logo - -

Hi {recipient_email},
- - I'm happy to provide you with an OpenAI Proxy API Key, loaded with ${key_budget} per month.

- - - Key:

{key_token}

-
- -

Usage Example

- - Detailed Documentation on Usage with OpenAI Python SDK, Langchain, LlamaIndex, Curl - -
-
-            import openai
-            client = openai.OpenAI(
-                api_key="{key_token}",
-                base_url={os.getenv("PROXY_BASE_URL", "http://0.0.0.0:4000")}
-            )
-
-            response = client.chat.completions.create(
-                model="gpt-3.5-turbo", # model to send to the proxy
-                messages = [
-                    {{
-                        "role": "user",
-                        "content": "this is a test request, write a short poem"
-                    }}
-                ]
-            )
-
-            
- - - If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT}

- - Best,
- The LiteLLM team
- """ - - payload = webhook_event.model_dump_json() - email_event = { - "to": recipient_email, - "subject": f"LiteLLM: {event_name}", - "html": email_html_content, - } - - response = await send_email( - receiver_email=email_event["to"], - subject=email_event["subject"], - html=email_event["html"], - ) - - return False + except Exception as e: + verbose_proxy_logger.error("Error sending email alert %s", str(e)) + return False async def send_email_alert_using_smtp(self, webhook_event: WebhookEvent) -> bool: """ @@ -1254,6 +1307,21 @@ Model Info: """ from litellm.proxy.utils import send_email + from litellm.proxy.proxy_server import premium_user, prisma_client + + email_logo_url = os.getenv( + "SMTP_SENDER_LOGO", os.getenv("EMAIL_LOGO_URL", None) + ) + email_support_contact = os.getenv("EMAIL_SUPPORT_CONTACT", None) + await self._check_if_using_premium_email_feature( + premium_user, email_logo_url, email_support_contact + ) + + if email_logo_url is None: + email_logo_url = LITELLM_LOGO_URL + if email_support_contact is None: + email_support_contact = LITELLM_SUPPORT_CONTACT + event_name = webhook_event.event_message recipient_email = webhook_event.user_email user_name = webhook_event.user_id @@ -1266,7 +1334,7 @@ Model Info: if webhook_event.event == "budget_crossed": email_html_content = f""" - LiteLLM Logo + LiteLLM Logo

Hi {user_name},
@@ -1274,7 +1342,7 @@ Model Info: API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month.

- If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT}

+ If you have any questions, please send an email to {email_support_contact}

Best,
The LiteLLM team
@@ -1384,7 +1452,9 @@ Model Info: if response.status_code == 200: pass else: - print("Error sending slack alert. Error=", response.text) # noqa + verbose_proxy_logger.debug( + "Error sending slack alert. Error=", response.text + ) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): """Log deployment latency""" @@ -1404,6 +1474,8 @@ Model Info: final_value = float( response_s.total_seconds() / completion_tokens ) + if isinstance(final_value, timedelta): + final_value = final_value.total_seconds() await self.async_update_daily_reports( DeploymentMetrics( diff --git a/litellm/integrations/supabase.py b/litellm/integrations/supabase.py index 4e6bf517f..7309342e4 100644 --- a/litellm/integrations/supabase.py +++ b/litellm/integrations/supabase.py @@ -110,6 +110,5 @@ class Supabase: ) except: - # traceback.print_exc() print_verbose(f"Supabase Logging Error - {traceback.format_exc()}") pass diff --git a/litellm/integrations/test_httpx.py b/litellm/integrations/test_httpx.py new file mode 100644 index 000000000..e69de29bb diff --git a/litellm/integrations/traceloop.py b/litellm/integrations/traceloop.py index bbdb9a1b0..e1c419c6f 100644 --- a/litellm/integrations/traceloop.py +++ b/litellm/integrations/traceloop.py @@ -1,114 +1,149 @@ +import traceback +from litellm._logging import verbose_logger +import litellm + + class TraceloopLogger: def __init__(self): - from traceloop.sdk.tracing.tracing import TracerWrapper - from traceloop.sdk import Traceloop + try: + from traceloop.sdk.tracing.tracing import TracerWrapper + from traceloop.sdk import Traceloop + from traceloop.sdk.instruments import Instruments + from opentelemetry.sdk.trace.export import ConsoleSpanExporter + except ModuleNotFoundError as e: + verbose_logger.error( + f"Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: {e}\n{traceback.format_exc()}" + ) - Traceloop.init(app_name="Litellm-Server", disable_batch=True) + Traceloop.init( + app_name="Litellm-Server", + disable_batch=True, + ) self.tracer_wrapper = TracerWrapper() - def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): - from opentelemetry.trace import SpanKind + def log_event( + self, + kwargs, + response_obj, + start_time, + end_time, + user_id, + print_verbose, + level="DEFAULT", + status_message=None, + ): + from opentelemetry import trace + from opentelemetry.trace import SpanKind, Status, StatusCode from opentelemetry.semconv.ai import SpanAttributes try: + print_verbose( + f"Traceloop Logging - Enters logging function for model {kwargs}" + ) + tracer = self.tracer_wrapper.get_tracer() - model = kwargs.get("model") - - # LiteLLM uses the standard OpenAI library, so it's already handled by Traceloop SDK - if kwargs.get("litellm_params").get("custom_llm_provider") == "openai": - return - optional_params = kwargs.get("optional_params", {}) - with tracer.start_as_current_span( - "litellm.completion", - kind=SpanKind.CLIENT, - ) as span: - if span.is_recording(): + start_time = int(start_time.timestamp()) + end_time = int(end_time.timestamp()) + span = tracer.start_span( + "litellm.completion", kind=SpanKind.CLIENT, start_time=start_time + ) + + if span.is_recording(): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model") + ) + if "stop" in optional_params: span.set_attribute( - SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model") + SpanAttributes.LLM_CHAT_STOP_SEQUENCES, + optional_params.get("stop"), ) - if "stop" in optional_params: - span.set_attribute( - SpanAttributes.LLM_CHAT_STOP_SEQUENCES, - optional_params.get("stop"), - ) - if "frequency_penalty" in optional_params: - span.set_attribute( - SpanAttributes.LLM_FREQUENCY_PENALTY, - optional_params.get("frequency_penalty"), - ) - if "presence_penalty" in optional_params: - span.set_attribute( - SpanAttributes.LLM_PRESENCE_PENALTY, - optional_params.get("presence_penalty"), - ) - if "top_p" in optional_params: - span.set_attribute( - SpanAttributes.LLM_TOP_P, optional_params.get("top_p") - ) - if "tools" in optional_params or "functions" in optional_params: - span.set_attribute( - SpanAttributes.LLM_REQUEST_FUNCTIONS, - optional_params.get( - "tools", optional_params.get("functions") - ), - ) - if "user" in optional_params: - span.set_attribute( - SpanAttributes.LLM_USER, optional_params.get("user") - ) - if "max_tokens" in optional_params: - span.set_attribute( - SpanAttributes.LLM_REQUEST_MAX_TOKENS, - kwargs.get("max_tokens"), - ) - if "temperature" in optional_params: - span.set_attribute( - SpanAttributes.LLM_TEMPERATURE, kwargs.get("temperature") - ) - - for idx, prompt in enumerate(kwargs.get("messages")): - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", - prompt.get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", - prompt.get("content"), - ) - + if "frequency_penalty" in optional_params: span.set_attribute( - SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model") + SpanAttributes.LLM_FREQUENCY_PENALTY, + optional_params.get("frequency_penalty"), + ) + if "presence_penalty" in optional_params: + span.set_attribute( + SpanAttributes.LLM_PRESENCE_PENALTY, + optional_params.get("presence_penalty"), + ) + if "top_p" in optional_params: + span.set_attribute( + SpanAttributes.LLM_TOP_P, optional_params.get("top_p") + ) + if "tools" in optional_params or "functions" in optional_params: + span.set_attribute( + SpanAttributes.LLM_REQUEST_FUNCTIONS, + optional_params.get("tools", optional_params.get("functions")), + ) + if "user" in optional_params: + span.set_attribute( + SpanAttributes.LLM_USER, optional_params.get("user") + ) + if "max_tokens" in optional_params: + span.set_attribute( + SpanAttributes.LLM_REQUEST_MAX_TOKENS, + kwargs.get("max_tokens"), + ) + if "temperature" in optional_params: + span.set_attribute( + SpanAttributes.LLM_REQUEST_TEMPERATURE, + kwargs.get("temperature"), ) - usage = response_obj.get("usage") - if usage: - span.set_attribute( - SpanAttributes.LLM_USAGE_TOTAL_TOKENS, - usage.get("total_tokens"), - ) - span.set_attribute( - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, - usage.get("completion_tokens"), - ) - span.set_attribute( - SpanAttributes.LLM_USAGE_PROMPT_TOKENS, - usage.get("prompt_tokens"), - ) - for idx, choice in enumerate(response_obj.get("choices")): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", - choice.get("finish_reason"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", - choice.get("message").get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", - choice.get("message").get("content"), - ) + for idx, prompt in enumerate(kwargs.get("messages")): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", + prompt.get("role"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", + prompt.get("content"), + ) + + span.set_attribute( + SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model") + ) + usage = response_obj.get("usage") + if usage: + span.set_attribute( + SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + usage.get("total_tokens"), + ) + span.set_attribute( + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + usage.get("completion_tokens"), + ) + span.set_attribute( + SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + usage.get("prompt_tokens"), + ) + + for idx, choice in enumerate(response_obj.get("choices")): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", + choice.get("finish_reason"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", + choice.get("message").get("role"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", + choice.get("message").get("content"), + ) + + if ( + level == "ERROR" + and status_message is not None + and isinstance(status_message, str) + ): + span.record_exception(Exception(status_message)) + span.set_status(Status(StatusCode.ERROR, status_message)) + + span.end(end_time) except Exception as e: print_verbose(f"Traceloop Layer Error - {e}") diff --git a/litellm/integrations/weights_biases.py b/litellm/integrations/weights_biases.py index a56233b22..1ac535c4f 100644 --- a/litellm/integrations/weights_biases.py +++ b/litellm/integrations/weights_biases.py @@ -217,6 +217,5 @@ class WeightsBiasesLogger: f"W&B Logging Logging - final response object: {response_obj}" ) except: - # traceback.print_exc() print_verbose(f"W&B Logging Layer Error - {traceback.format_exc()}") pass diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 1ca048523..8e469a8f4 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -3,6 +3,7 @@ import json from enum import Enum import requests, copy # type: ignore import time +from functools import partial from typing import Callable, Optional, List, Union from litellm.utils import ModelResponse, Usage, map_finish_reason, CustomStreamWrapper import litellm @@ -160,6 +161,36 @@ def validate_environment(api_key, user_headers): return headers +async def make_call( + client: Optional[AsyncHTTPHandler], + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj, +): + if client is None: + client = AsyncHTTPHandler() # Create a new client if none provided + + response = await client.post(api_base, headers=headers, data=data, stream=True) + + if response.status_code != 200: + raise AnthropicError(status_code=response.status_code, message=response.text) + + completion_stream = response.aiter_lines() + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response=completion_stream, # Pass the completion stream for logging + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + + class AnthropicChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() @@ -379,23 +410,34 @@ class AnthropicChatCompletion(BaseLLM): logger_fn=None, headers={}, ): - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) - ) data["stream"] = True - response = await self.async_handler.post( - api_base, headers=headers, data=json.dumps(data), stream=True - ) + # async_handler = AsyncHTTPHandler( + # timeout=httpx.Timeout(timeout=600.0, connect=20.0) + # ) - if response.status_code != 200: - raise AnthropicError( - status_code=response.status_code, message=response.text - ) + # response = await async_handler.post( + # api_base, headers=headers, json=data, stream=True + # ) - completion_stream = response.aiter_lines() + # if response.status_code != 200: + # raise AnthropicError( + # status_code=response.status_code, message=response.text + # ) + + # completion_stream = response.aiter_lines() streamwrapper = CustomStreamWrapper( - completion_stream=completion_stream, + completion_stream=None, + make_call=partial( + make_call, + client=None, + api_base=api_base, + headers=headers, + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + ), model=model, custom_llm_provider="anthropic", logging_obj=logging_obj, @@ -421,12 +463,10 @@ class AnthropicChatCompletion(BaseLLM): logger_fn=None, headers={}, ) -> Union[ModelResponse, CustomStreamWrapper]: - self.async_handler = AsyncHTTPHandler( + async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) - response = await self.async_handler.post( - api_base, headers=headers, data=json.dumps(data) - ) + response = await async_handler.post(api_base, headers=headers, json=data) if stream and _is_function_call: return self.process_streaming_response( model=model, diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index 02fe4a08f..834fcbea9 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -1,4 +1,5 @@ -from typing import Optional, Union, Any, Literal +from typing import Optional, Union, Any, Literal, Coroutine, Iterable +from typing_extensions import overload import types, requests from .base import BaseLLM from litellm.utils import ( @@ -9,6 +10,7 @@ from litellm.utils import ( convert_to_model_response_object, TranscriptionResponse, get_secret, + UnsupportedParamsError, ) from typing import Callable, Optional, BinaryIO, List from litellm import OpenAIConfig @@ -18,6 +20,22 @@ from .custom_httpx.azure_dall_e_2 import CustomHTTPTransport, AsyncCustomHTTPTra from openai import AzureOpenAI, AsyncAzureOpenAI import uuid import os +from ..types.llms.openai import ( + AsyncCursorPage, + AssistantToolParam, + SyncCursorPage, + Assistant, + MessageData, + OpenAIMessage, + OpenAICreateThreadParamsMessage, + Thread, + AssistantToolParam, + Run, + AssistantEventHandler, + AsyncAssistantEventHandler, + AsyncAssistantStreamManager, + AssistantStreamManager, +) class AzureOpenAIError(Exception): @@ -45,9 +63,9 @@ class AzureOpenAIError(Exception): ) # Call the base class constructor with the parameters it needs -class AzureOpenAIConfig(OpenAIConfig): +class AzureOpenAIConfig: """ - Reference: https://platform.openai.com/docs/api-reference/chat/create + Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions The class `AzureOpenAIConfig` provides configuration for the OpenAI's Chat API interface, for use with Azure. It inherits from `OpenAIConfig`. Below are the parameters:: @@ -85,18 +103,111 @@ class AzureOpenAIConfig(OpenAIConfig): temperature: Optional[int] = None, top_p: Optional[int] = None, ) -> None: - super().__init__( - frequency_penalty, - function_call, - functions, - logit_bias, - max_tokens, - n, - presence_penalty, - stop, - temperature, - top_p, - ) + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return [ + "temperature", + "n", + "stream", + "stop", + "max_tokens", + "tools", + "tool_choice", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "function_call", + "functions", + "tools", + "tool_choice", + "top_p", + "logprobs", + "top_logprobs", + "response_format", + "seed", + "extra_headers", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + api_version: str, # Y-M-D-{optional} + drop_params, + ) -> dict: + supported_openai_params = self.get_supported_openai_params() + + api_version_times = api_version.split("-") + api_version_year = api_version_times[0] + api_version_month = api_version_times[1] + api_version_day = api_version_times[2] + for param, value in non_default_params.items(): + if param == "tool_choice": + """ + This parameter requires API version 2023-12-01-preview or later + + tool_choice='required' is not supported as of 2024-05-01-preview + """ + ## check if api version supports this param ## + if ( + api_version_year < "2023" + or (api_version_year == "2023" and api_version_month < "12") + or ( + api_version_year == "2023" + and api_version_month == "12" + and api_version_day < "01" + ) + ): + if litellm.drop_params == True or ( + drop_params is not None and drop_params == True + ): + pass + else: + raise UnsupportedParamsError( + status_code=400, + message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""", + ) + elif value == "required" and ( + api_version_year == "2024" and api_version_month <= "05" + ): ## check if tool_choice value is supported ## + if litellm.drop_params == True or ( + drop_params is not None and drop_params == True + ): + pass + else: + raise UnsupportedParamsError( + status_code=400, + message=f"Azure does not support '{value}' as a {param} param, for api_version={api_version}. To drop 'tool_choice=required' for calls with this Azure API version, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\nAzure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions", + ) + else: + optional_params["tool_choice"] = value + elif param in supported_openai_params: + optional_params[param] = value + return optional_params def get_mapped_special_auth_params(self) -> dict: return {"token": "azure_ad_token"} @@ -114,6 +225,68 @@ class AzureOpenAIConfig(OpenAIConfig): return ["europe", "sweden", "switzerland", "france", "uk"] +class AzureOpenAIAssistantsAPIConfig: + """ + Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message + """ + + def __init__( + self, + ) -> None: + pass + + def get_supported_openai_create_message_params(self): + return [ + "role", + "content", + "attachments", + "metadata", + ] + + def map_openai_params_create_message_params( + self, non_default_params: dict, optional_params: dict + ): + for param, value in non_default_params.items(): + if param == "role": + optional_params["role"] = value + if param == "metadata": + optional_params["metadata"] = value + elif param == "content": # only string accepted + if isinstance(value, str): + optional_params["content"] = value + else: + raise litellm.utils.UnsupportedParamsError( + message="Azure only accepts content as a string.", + status_code=400, + ) + elif ( + param == "attachments" + ): # this is a v2 param. Azure currently supports the old 'file_id's param + file_ids: List[str] = [] + if isinstance(value, list): + for item in value: + if "file_id" in item: + file_ids.append(item["file_id"]) + else: + if litellm.drop_params == True: + pass + else: + raise litellm.utils.UnsupportedParamsError( + message="Azure doesn't support {}. To drop it from the call, set `litellm.drop_params = True.".format( + value + ), + status_code=400, + ) + else: + raise litellm.utils.UnsupportedParamsError( + message="Invalid param. attachments should always be a list. Got={}, Expected=List. Raw value={}".format( + type(value), value + ), + status_code=400, + ) + return optional_params + + def select_azure_base_url_or_endpoint(azure_client_params: dict): # azure_client_params = { # "api_version": api_version, @@ -172,9 +345,7 @@ def get_azure_ad_token_from_oidc(azure_ad_token: str): possible_azure_ad_token = req_token.json().get("access_token", None) if possible_azure_ad_token is None: - raise AzureOpenAIError( - status_code=422, message="Azure AD Token not returned" - ) + raise AzureOpenAIError(status_code=422, message="Azure AD Token not returned") return possible_azure_ad_token @@ -245,7 +416,9 @@ class AzureChatCompletion(BaseLLM): azure_client_params["api_key"] = api_key elif azure_ad_token is not None: if azure_ad_token.startswith("oidc/"): - azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) + azure_ad_token = get_azure_ad_token_from_oidc( + azure_ad_token + ) azure_client_params["azure_ad_token"] = azure_ad_token @@ -1192,3 +1365,828 @@ class AzureChatCompletion(BaseLLM): response["x-ms-region"] = completion.headers["x-ms-region"] return response + + +class AzureAssistantsAPI(BaseLLM): + def __init__(self) -> None: + super().__init__() + + def get_azure_client( + self, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AzureOpenAI] = None, + ) -> AzureOpenAI: + received_args = locals() + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client": + pass + elif k == "api_base" and v is not None: + data["azure_endpoint"] = v + elif v is not None: + data[k] = v + azure_openai_client = AzureOpenAI(**data) # type: ignore + else: + azure_openai_client = client + + return azure_openai_client + + def async_get_azure_client( + self, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI] = None, + ) -> AsyncAzureOpenAI: + received_args = locals() + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client": + pass + elif k == "api_base" and v is not None: + data["azure_endpoint"] = v + elif v is not None: + data[k] = v + + azure_openai_client = AsyncAzureOpenAI(**data) + # azure_openai_client = AsyncAzureOpenAI(**data) # type: ignore + else: + azure_openai_client = client + + return azure_openai_client + + ### ASSISTANTS ### + + async def async_get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + ) -> AsyncCursorPage[Assistant]: + azure_openai_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = await azure_openai_client.beta.assistants.list() + + return response + + # fmt: off + + @overload + def get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + aget_assistants: Literal[True], + ) -> Coroutine[None, None, AsyncCursorPage[Assistant]]: + ... + + @overload + def get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AzureOpenAI], + aget_assistants: Optional[Literal[False]], + ) -> SyncCursorPage[Assistant]: + ... + + # fmt: on + + def get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client=None, + aget_assistants=None, + ): + if aget_assistants is not None and aget_assistants == True: + return self.async_get_assistants( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + azure_openai_client = self.get_azure_client( + api_key=api_key, + api_base=api_base, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + api_version=api_version, + ) + + response = azure_openai_client.beta.assistants.list() + + return response + + ### MESSAGES ### + + async def a_add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI] = None, + ) -> OpenAIMessage: + openai_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + thread_message: OpenAIMessage = await openai_client.beta.threads.messages.create( # type: ignore + thread_id, **message_data # type: ignore + ) + + response_obj: Optional[OpenAIMessage] = None + if getattr(thread_message, "status", None) is None: + thread_message.status = "completed" + response_obj = OpenAIMessage(**thread_message.dict()) + else: + response_obj = OpenAIMessage(**thread_message.dict()) + return response_obj + + # fmt: off + + @overload + def add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + a_add_message: Literal[True], + ) -> Coroutine[None, None, OpenAIMessage]: + ... + + @overload + def add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AzureOpenAI], + a_add_message: Optional[Literal[False]], + ) -> OpenAIMessage: + ... + + # fmt: on + + def add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client=None, + a_add_message: Optional[bool] = None, + ): + if a_add_message is not None and a_add_message == True: + return self.a_add_message( + thread_id=thread_id, + message_data=message_data, + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + openai_client = self.get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + thread_message: OpenAIMessage = openai_client.beta.threads.messages.create( # type: ignore + thread_id, **message_data # type: ignore + ) + + response_obj: Optional[OpenAIMessage] = None + if getattr(thread_message, "status", None) is None: + thread_message.status = "completed" + response_obj = OpenAIMessage(**thread_message.dict()) + else: + response_obj = OpenAIMessage(**thread_message.dict()) + return response_obj + + async def async_get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI] = None, + ) -> AsyncCursorPage[OpenAIMessage]: + openai_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = await openai_client.beta.threads.messages.list(thread_id=thread_id) + + return response + + # fmt: off + + @overload + def get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + aget_messages: Literal[True], + ) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]: + ... + + @overload + def get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AzureOpenAI], + aget_messages: Optional[Literal[False]], + ) -> SyncCursorPage[OpenAIMessage]: + ... + + # fmt: on + + def get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client=None, + aget_messages=None, + ): + if aget_messages is not None and aget_messages == True: + return self.async_get_messages( + thread_id=thread_id, + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + openai_client = self.get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = openai_client.beta.threads.messages.list(thread_id=thread_id) + + return response + + ### THREADS ### + + async def async_create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + ) -> Thread: + openai_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + data = {} + if messages is not None: + data["messages"] = messages # type: ignore + if metadata is not None: + data["metadata"] = metadata # type: ignore + + message_thread = await openai_client.beta.threads.create(**data) # type: ignore + + return Thread(**message_thread.dict()) + + # fmt: off + + @overload + def create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + client: Optional[AsyncAzureOpenAI], + acreate_thread: Literal[True], + ) -> Coroutine[None, None, Thread]: + ... + + @overload + def create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + client: Optional[AzureOpenAI], + acreate_thread: Optional[Literal[False]], + ) -> Thread: + ... + + # fmt: on + + def create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + client=None, + acreate_thread=None, + ): + """ + Here's an example: + ``` + from litellm.llms.openai import OpenAIAssistantsAPI, MessageData + + # create thread + message: MessageData = {"role": "user", "content": "Hey, how's it going?"} + openai_api.create_thread(messages=[message]) + ``` + """ + if acreate_thread is not None and acreate_thread == True: + return self.async_create_thread( + metadata=metadata, + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + messages=messages, + ) + azure_openai_client = self.get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + data = {} + if messages is not None: + data["messages"] = messages # type: ignore + if metadata is not None: + data["metadata"] = metadata # type: ignore + + message_thread = azure_openai_client.beta.threads.create(**data) # type: ignore + + return Thread(**message_thread.dict()) + + async def async_get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + ) -> Thread: + openai_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = await openai_client.beta.threads.retrieve(thread_id=thread_id) + + return Thread(**response.dict()) + + # fmt: off + + @overload + def get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + aget_thread: Literal[True], + ) -> Coroutine[None, None, Thread]: + ... + + @overload + def get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AzureOpenAI], + aget_thread: Optional[Literal[False]], + ) -> Thread: + ... + + # fmt: on + + def get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client=None, + aget_thread=None, + ): + if aget_thread is not None and aget_thread == True: + return self.async_get_thread( + thread_id=thread_id, + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + openai_client = self.get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = openai_client.beta.threads.retrieve(thread_id=thread_id) + + return Thread(**response.dict()) + + # def delete_thread(self): + # pass + + ### RUNS ### + + async def arun_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + ) -> Run: + openai_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + api_version=api_version, + azure_ad_token=azure_ad_token, + client=client, + ) + + response = await openai_client.beta.threads.runs.create_and_poll( # type: ignore + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + ) + + return response + + def async_run_thread_stream( + self, + client: AsyncAzureOpenAI, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + tools: Optional[Iterable[AssistantToolParam]], + event_handler: Optional[AssistantEventHandler], + ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: + data = { + "thread_id": thread_id, + "assistant_id": assistant_id, + "additional_instructions": additional_instructions, + "instructions": instructions, + "metadata": metadata, + "model": model, + "tools": tools, + } + if event_handler is not None: + data["event_handler"] = event_handler + return client.beta.threads.runs.stream(**data) # type: ignore + + def run_thread_stream( + self, + client: AzureOpenAI, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + tools: Optional[Iterable[AssistantToolParam]], + event_handler: Optional[AssistantEventHandler], + ) -> AssistantStreamManager[AssistantEventHandler]: + data = { + "thread_id": thread_id, + "assistant_id": assistant_id, + "additional_instructions": additional_instructions, + "instructions": instructions, + "metadata": metadata, + "model": model, + "tools": tools, + } + if event_handler is not None: + data["event_handler"] = event_handler + return client.beta.threads.runs.stream(**data) # type: ignore + + # fmt: off + + @overload + def run_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AsyncAzureOpenAI], + arun_thread: Literal[True], + ) -> Coroutine[None, None, Run]: + ... + + @overload + def run_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[AzureOpenAI], + arun_thread: Optional[Literal[False]], + ) -> Run: + ... + + # fmt: on + + def run_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + api_version: Optional[str], + azure_ad_token: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client=None, + arun_thread=None, + event_handler: Optional[AssistantEventHandler] = None, + ): + if arun_thread is not None and arun_thread == True: + if stream is not None and stream == True: + azure_client = self.async_get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + return self.async_run_thread_stream( + client=azure_client, + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + event_handler=event_handler, + ) + return self.arun_thread( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + stream=stream, + tools=tools, + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + openai_client = self.get_azure_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + azure_ad_token=azure_ad_token, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + if stream is not None and stream == True: + return self.run_thread_stream( + client=openai_client, + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + event_handler=event_handler, + ) + + response = openai_client.beta.threads.runs.create_and_poll( # type: ignore + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + ) + + return response diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py index 337055dc2..b011d9512 100644 --- a/litellm/llms/bedrock_httpx.py +++ b/litellm/llms/bedrock_httpx.py @@ -1,7 +1,7 @@ # What is this? ## Initial implementation of calling bedrock via httpx client (allows for async calls). ## V1 - covers cohere + anthropic claude-3 support - +from functools import partial import os, types import json from enum import Enum @@ -38,6 +38,8 @@ from .prompt_templates.factory import ( extract_between_tags, parse_xml_params, contains_tag, + _bedrock_converse_messages_pt, + _bedrock_tools_pt, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from .base import BaseLLM @@ -45,6 +47,12 @@ import httpx # type: ignore from .bedrock import BedrockError, convert_messages_to_prompt, ModelResponseIterator from litellm.types.llms.bedrock import * import urllib.parse +from litellm.types.llms.openai import ( + ChatCompletionResponseMessage, + ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, + ChatCompletionDeltaChunk, +) class AmazonCohereChatConfig: @@ -118,6 +126,8 @@ class AmazonCohereChatConfig: "presence_penalty", "seed", "stop", + "tools", + "tool_choice", ] def map_openai_params( @@ -145,6 +155,68 @@ class AmazonCohereChatConfig: return optional_params +async def make_call( + client: Optional[AsyncHTTPHandler], + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj, +): + if client is None: + client = AsyncHTTPHandler() # Create a new client if none provided + + response = await client.post(api_base, headers=headers, data=data, stream=True) + + if response.status_code != 200: + raise BedrockError(status_code=response.status_code, message=response.text) + + decoder = AWSEventStreamDecoder(model=model) + completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024)) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + + +def make_sync_call( + client: Optional[HTTPHandler], + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj, +): + if client is None: + client = HTTPHandler() # Create a new client if none provided + + response = client.post(api_base, headers=headers, data=data, stream=True) + + if response.status_code != 200: + raise BedrockError(status_code=response.status_code, message=response.read()) + + decoder = AWSEventStreamDecoder(model=model) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + + class BedrockLLM(BaseLLM): """ Example call @@ -217,6 +289,7 @@ class BedrockLLM(BaseLLM): aws_session_name: Optional[str] = None, aws_profile_name: Optional[str] = None, aws_role_name: Optional[str] = None, + aws_web_identity_token: Optional[str] = None, ): """ Return a boto3.Credentials object @@ -231,6 +304,7 @@ class BedrockLLM(BaseLLM): aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ] # Iterate over parameters and update if needed @@ -247,10 +321,43 @@ class BedrockLLM(BaseLLM): aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ) = params_to_check ### CHECK STS ### - if aws_role_name is not None and aws_session_name is not None: + if ( + aws_web_identity_token is not None + and aws_role_name is not None + and aws_session_name is not None + ): + oidc_token = get_secret(aws_web_identity_token) + + if oidc_token is None: + raise BedrockError( + message="OIDC token could not be retrieved from secret manager.", + status_code=401, + ) + + sts_client = boto3.client("sts") + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html + sts_response = sts_client.assume_role_with_web_identity( + RoleArn=aws_role_name, + RoleSessionName=aws_session_name, + WebIdentityToken=oidc_token, + DurationSeconds=3600, + ) + + session = boto3.Session( + aws_access_key_id=sts_response["Credentials"]["AccessKeyId"], + aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"], + aws_session_token=sts_response["Credentials"]["SessionToken"], + region_name=aws_region_name, + ) + + return session.get_credentials() + elif aws_role_name is not None and aws_session_name is not None: sts_client = boto3.client( "sts", aws_access_key_id=aws_access_key_id, # [OPTIONAL] @@ -261,7 +368,16 @@ class BedrockLLM(BaseLLM): RoleArn=aws_role_name, RoleSessionName=aws_session_name ) - return sts_response["Credentials"] + # Extract the credentials from the response and convert to Session Credentials + sts_credentials = sts_response["Credentials"] + from botocore.credentials import Credentials + + credentials = Credentials( + access_key=sts_credentials["AccessKeyId"], + secret_key=sts_credentials["SecretAccessKey"], + token=sts_credentials["SessionToken"], + ) + return credentials elif aws_profile_name is not None: ### CHECK SESSION ### # uses auth values from AWS profile usually stored in ~/.aws/credentials client = boto3.Session(profile_name=aws_profile_name) @@ -582,6 +698,7 @@ class BedrockLLM(BaseLLM): aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) # https://bedrock-runtime.{region_name}.amazonaws.com + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) ### SET REGION NAME ### if aws_region_name is None: @@ -609,6 +726,7 @@ class BedrockLLM(BaseLLM): aws_session_name=aws_session_name, aws_profile_name=aws_profile_name, aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, ) ### SET RUNTIME ENDPOINT ### @@ -923,16 +1041,16 @@ class BedrockLLM(BaseLLM): if isinstance(timeout, float) or isinstance(timeout, int): timeout = httpx.Timeout(timeout) _params["timeout"] = timeout - self.client = AsyncHTTPHandler(**_params) # type: ignore + client = AsyncHTTPHandler(**_params) # type: ignore else: - self.client = client # type: ignore + client = client # type: ignore try: - response = await self.client.post(api_base, headers=headers, data=data) # type: ignore + response = await client.post(api_base, headers=headers, data=data) # type: ignore response.raise_for_status() except httpx.HTTPStatusError as err: error_code = err.response.status_code - raise BedrockError(status_code=error_code, message=response.text) + raise BedrockError(status_code=error_code, message=err.response.text) except httpx.TimeoutException as e: raise BedrockError(status_code=408, message="Timeout error occurred.") @@ -968,43 +1086,760 @@ class BedrockLLM(BaseLLM): headers={}, client: Optional[AsyncHTTPHandler] = None, ) -> CustomStreamWrapper: + # The call is not made here; instead, we prepare the necessary objects for the stream. + + streaming_response = CustomStreamWrapper( + completion_stream=None, + make_call=partial( + make_call, + client=client, + api_base=api_base, + headers=headers, + data=data, + model=model, + messages=messages, + logging_obj=logging_obj, + ), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + return streaming_response + + def embedding(self, *args, **kwargs): + return super().embedding(*args, **kwargs) + + +class AmazonConverseConfig: + """ + Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html + #2 - https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html#conversation-inference-supported-models-features + """ + + maxTokens: Optional[int] + stopSequences: Optional[List[str]] + temperature: Optional[int] + topP: Optional[int] + + def __init__( + self, + maxTokens: Optional[int] = None, + stopSequences: Optional[List[str]] = None, + temperature: Optional[int] = None, + topP: Optional[int] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self, model: str) -> List[str]: + supported_params = [ + "max_tokens", + "stream", + "stream_options", + "stop", + "temperature", + "top_p", + "extra_headers", + ] + + if ( + model.startswith("anthropic") + or model.startswith("mistral") + or model.startswith("cohere") + ): + supported_params.append("tools") + + if model.startswith("anthropic") or model.startswith("mistral"): + # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html + supported_params.append("tool_choice") + + return supported_params + + def map_tool_choice_values( + self, model: str, tool_choice: Union[str, dict], drop_params: bool + ) -> Optional[ToolChoiceValuesBlock]: + if tool_choice == "none": + if litellm.drop_params is True or drop_params is True: + return None + else: + raise litellm.utils.UnsupportedParamsError( + message="Bedrock doesn't support tool_choice={}. To drop it from the call, set `litellm.drop_params = True.".format( + tool_choice + ), + status_code=400, + ) + elif tool_choice == "required": + return ToolChoiceValuesBlock(any={}) + elif tool_choice == "auto": + return ToolChoiceValuesBlock(auto={}) + elif isinstance(tool_choice, dict): + # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html + specific_tool = SpecificToolChoiceBlock( + name=tool_choice.get("function", {}).get("name", "") + ) + return ToolChoiceValuesBlock(tool=specific_tool) + else: + raise litellm.utils.UnsupportedParamsError( + message="Bedrock doesn't support tool_choice={}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.".format( + tool_choice + ), + status_code=400, + ) + + def get_supported_image_types(self) -> List[str]: + return ["png", "jpeg", "gif", "webp"] + + def map_openai_params( + self, + model: str, + non_default_params: dict, + optional_params: dict, + drop_params: bool, + ) -> dict: + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["maxTokens"] = value + if param == "stream": + optional_params["stream"] = value + if param == "stop": + if isinstance(value, str): + value = [value] + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["topP"] = value + if param == "tools": + optional_params["tools"] = value + if param == "tool_choice": + _tool_choice_value = self.map_tool_choice_values( + model=model, tool_choice=value, drop_params=drop_params # type: ignore + ) + if _tool_choice_value is not None: + optional_params["tool_choice"] = _tool_choice_value + return optional_params + + +class BedrockConverseLLM(BaseLLM): + def __init__(self) -> None: + super().__init__() + + def process_response( + self, + model: str, + response: Union[requests.Response, httpx.Response], + model_response: ModelResponse, + stream: bool, + logging_obj: Logging, + optional_params: dict, + api_key: str, + data: Union[dict, str], + messages: List, + print_verbose, + encoding, + ) -> Union[ModelResponse, CustomStreamWrapper]: + + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + + ## RESPONSE OBJECT + try: + completion_response = ConverseResponseBlock(**response.json()) # type: ignore + except Exception as e: + raise BedrockError( + message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( + response.text, str(e) + ), + status_code=422, + ) + + """ + Bedrock Response Object has optional message block + + completion_response["output"].get("message", None) + + A message block looks like this (Example 1): + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "Is there anything else you'd like to talk about? Perhaps I can help with some economic questions or provide some information about economic concepts?" + } + ] + } + }, + (Example 2): + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_hbTgdi0CSLq_hM4P8csZJA", + "name": "top_song", + "input": { + "sign": "WZPZ" + } + } + } + ] + } + } + + """ + message: Optional[MessageBlock] = completion_response["output"]["message"] + chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} + content_str = "" + tools: List[ChatCompletionToolCallChunk] = [] + if message is not None: + for content in message["content"]: + """ + - Content is either a tool response or text + """ + if "text" in content: + content_str += content["text"] + if "toolUse" in content: + _function_chunk = ChatCompletionToolCallFunctionChunk( + name=content["toolUse"]["name"], + arguments=json.dumps(content["toolUse"]["input"]), + ) + _tool_response_chunk = ChatCompletionToolCallChunk( + id=content["toolUse"]["toolUseId"], + type="function", + function=_function_chunk, + ) + tools.append(_tool_response_chunk) + chat_completion_message["content"] = content_str + chat_completion_message["tool_calls"] = tools + + ## CALCULATING USAGE - bedrock returns usage in the headers + input_tokens = completion_response["usage"]["inputTokens"] + output_tokens = completion_response["usage"]["outputTokens"] + total_tokens = completion_response["usage"]["totalTokens"] + + model_response.choices = [ + litellm.Choices( + finish_reason=map_finish_reason(completion_response["stopReason"]), + index=0, + message=litellm.Message(**chat_completion_message), + ) + ] + model_response["created"] = int(time.time()) + model_response["model"] = model + usage = Usage( + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + total_tokens=total_tokens, + ) + setattr(model_response, "usage", usage) + + return model_response + + def encode_model_id(self, model_id: str) -> str: + """ + Double encode the model ID to ensure it matches the expected double-encoded format. + Args: + model_id (str): The model ID to encode. + Returns: + str: The double-encoded model ID. + """ + return urllib.parse.quote(model_id, safe="") + + def get_credentials( + self, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_region_name: Optional[str] = None, + aws_session_name: Optional[str] = None, + aws_profile_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + aws_web_identity_token: Optional[str] = None, + ): + """ + Return a boto3.Credentials object + """ + import boto3 + + ## CHECK IS 'os.environ/' passed in + params_to_check: List[Optional[str]] = [ + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + ] + + # Iterate over parameters and update if needed + for i, param in enumerate(params_to_check): + if param and param.startswith("os.environ/"): + _v = get_secret(param) + if _v is not None and isinstance(_v, str): + params_to_check[i] = _v + # Assign updated values back to parameters + ( + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + aws_web_identity_token, + ) = params_to_check + + ### CHECK STS ### + if ( + aws_web_identity_token is not None + and aws_role_name is not None + and aws_session_name is not None + ): + oidc_token = get_secret(aws_web_identity_token) + + if oidc_token is None: + raise BedrockError( + message="OIDC token could not be retrieved from secret manager.", + status_code=401, + ) + + sts_client = boto3.client("sts") + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html + sts_response = sts_client.assume_role_with_web_identity( + RoleArn=aws_role_name, + RoleSessionName=aws_session_name, + WebIdentityToken=oidc_token, + DurationSeconds=3600, + ) + + session = boto3.Session( + aws_access_key_id=sts_response["Credentials"]["AccessKeyId"], + aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"], + aws_session_token=sts_response["Credentials"]["SessionToken"], + region_name=aws_region_name, + ) + + return session.get_credentials() + elif aws_role_name is not None and aws_session_name is not None: + sts_client = boto3.client( + "sts", + aws_access_key_id=aws_access_key_id, # [OPTIONAL] + aws_secret_access_key=aws_secret_access_key, # [OPTIONAL] + ) + + sts_response = sts_client.assume_role( + RoleArn=aws_role_name, RoleSessionName=aws_session_name + ) + + # Extract the credentials from the response and convert to Session Credentials + sts_credentials = sts_response["Credentials"] + from botocore.credentials import Credentials + + credentials = Credentials( + access_key=sts_credentials["AccessKeyId"], + secret_key=sts_credentials["SecretAccessKey"], + token=sts_credentials["SessionToken"], + ) + return credentials + elif aws_profile_name is not None: ### CHECK SESSION ### + # uses auth values from AWS profile usually stored in ~/.aws/credentials + client = boto3.Session(profile_name=aws_profile_name) + + return client.get_credentials() + else: + session = boto3.Session( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region_name, + ) + + return session.get_credentials() + + async def async_streaming( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + data: str, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj, + stream, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + ) -> CustomStreamWrapper: + streaming_response = CustomStreamWrapper( + completion_stream=None, + make_call=partial( + make_call, + client=client, + api_base=api_base, + headers=headers, + data=data, + model=model, + messages=messages, + logging_obj=logging_obj, + ), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + return streaming_response + + async def async_completion( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + data: str, + timeout: Optional[Union[float, httpx.Timeout]], + encoding, + logging_obj, + stream, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + client: Optional[AsyncHTTPHandler] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: if client is None: _params = {} if timeout is not None: if isinstance(timeout, float) or isinstance(timeout, int): timeout = httpx.Timeout(timeout) _params["timeout"] = timeout - self.client = AsyncHTTPHandler(**_params) # type: ignore + client = AsyncHTTPHandler(**_params) # type: ignore else: - self.client = client # type: ignore + client = client # type: ignore - response = await self.client.post(api_base, headers=headers, data=data, stream=True) # type: ignore + try: + response = await client.post(api_base, headers=headers, data=data) # type: ignore + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException as e: + raise BedrockError(status_code=408, message="Timeout error occurred.") - if response.status_code != 200: - raise BedrockError(status_code=response.status_code, message=response.text) - - decoder = AWSEventStreamDecoder(model=model) - - completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024)) - streaming_response = CustomStreamWrapper( - completion_stream=completion_stream, + return self.process_response( model=model, - custom_llm_provider="bedrock", + response=response, + model_response=model_response, + stream=stream if isinstance(stream, bool) else False, logging_obj=logging_obj, + api_key="", + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, ) + def completion( + self, + model: str, + messages: list, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj, + optional_params: dict, + acompletion: bool, + timeout: Optional[Union[float, httpx.Timeout]], + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None, + ): + try: + import boto3 + + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + except ImportError: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + ## SETUP ## + stream = optional_params.pop("stream", None) + modelId = optional_params.pop("model_id", None) + if modelId is not None: + modelId = self.encode_model_id(model_id=modelId) + else: + modelId = model + + provider = model.split(".")[0] + + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_region_name = optional_params.pop("aws_region_name", None) + aws_role_name = optional_params.pop("aws_role_name", None) + aws_session_name = optional_params.pop("aws_session_name", None) + aws_profile_name = optional_params.pop("aws_profile_name", None) + aws_bedrock_runtime_endpoint = optional_params.pop( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) + + ### SET REGION NAME ### + if aws_region_name is None: + # check env # + litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) + + if litellm_aws_region_name is not None and isinstance( + litellm_aws_region_name, str + ): + aws_region_name = litellm_aws_region_name + + standard_aws_region_name = get_secret("AWS_REGION", None) + if standard_aws_region_name is not None and isinstance( + standard_aws_region_name, str + ): + aws_region_name = standard_aws_region_name + + if aws_region_name is None: + aws_region_name = "us-west-2" + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + aws_web_identity_token=aws_web_identity_token, + ) + + ### SET RUNTIME ENDPOINT ### + endpoint_url = "" + env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT") + if aws_bedrock_runtime_endpoint is not None and isinstance( + aws_bedrock_runtime_endpoint, str + ): + endpoint_url = aws_bedrock_runtime_endpoint + elif env_aws_bedrock_runtime_endpoint and isinstance( + env_aws_bedrock_runtime_endpoint, str + ): + endpoint_url = env_aws_bedrock_runtime_endpoint + else: + endpoint_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" + + if (stream is not None and stream is True) and provider != "ai21": + endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream" + else: + endpoint_url = f"{endpoint_url}/model/{modelId}/converse" + + sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) + + # Separate system prompt from rest of message + system_prompt_indices = [] + system_content_blocks: List[SystemContentBlock] = [] + for idx, message in enumerate(messages): + if message["role"] == "system": + _system_content_block = SystemContentBlock(text=message["content"]) + system_content_blocks.append(_system_content_block) + system_prompt_indices.append(idx) + if len(system_prompt_indices) > 0: + for idx in reversed(system_prompt_indices): + messages.pop(idx) + + inference_params = copy.deepcopy(optional_params) + additional_request_keys = [] + additional_request_params = {} + supported_converse_params = AmazonConverseConfig.__annotations__.keys() + supported_tool_call_params = ["tools", "tool_choice"] + ## TRANSFORMATION ## + # send all model-specific params in 'additional_request_params' + for k, v in inference_params.items(): + if ( + k not in supported_converse_params + and k not in supported_tool_call_params + ): + additional_request_params[k] = v + additional_request_keys.append(k) + for key in additional_request_keys: + inference_params.pop(key, None) + + bedrock_messages: List[MessageBlock] = _bedrock_converse_messages_pt( + messages=messages + ) + bedrock_tools: List[ToolBlock] = _bedrock_tools_pt( + inference_params.pop("tools", []) + ) + bedrock_tool_config: Optional[ToolConfigBlock] = None + if len(bedrock_tools) > 0: + tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( + "tool_choice", None + ) + bedrock_tool_config = ToolConfigBlock( + tools=bedrock_tools, + ) + if tool_choice_values is not None: + bedrock_tool_config["toolChoice"] = tool_choice_values + + _data: RequestObject = { + "messages": bedrock_messages, + "additionalModelRequestFields": additional_request_params, + "system": system_content_blocks, + "inferenceConfig": InferenceConfig(**inference_params), + } + if bedrock_tool_config is not None: + _data["toolConfig"] = bedrock_tool_config + data = json.dumps(_data) + ## COMPLETION CALL + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + request = AWSRequest( + method="POST", url=endpoint_url, data=data, headers=headers + ) + sigv4.add_auth(request) + prepped = request.prepare() + ## LOGGING - logging_obj.post_call( + logging_obj.pre_call( input=messages, api_key="", - original_response=streaming_response, - additional_args={"complete_input_dict": data}, + additional_args={ + "complete_input_dict": data, + "api_base": prepped.url, + "headers": prepped.headers, + }, ) - return streaming_response + ### ROUTING (ASYNC, STREAMING, SYNC) + if acompletion: + if isinstance(client, HTTPHandler): + client = None + if stream is True and provider != "ai21": + return self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=prepped.url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) # type: ignore + ### ASYNC COMPLETION + return self.async_completion( + model=model, + messages=messages, + data=data, + api_base=prepped.url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) # type: ignore - def embedding(self, *args, **kwargs): - return super().embedding(*args, **kwargs) + if (stream is not None and stream is True) and provider != "ai21": + + streaming_response = CustomStreamWrapper( + completion_stream=None, + make_call=partial( + make_sync_call, + client=None, + api_base=prepped.url, + headers=prepped.headers, # type: ignore + data=data, + model=model, + messages=messages, + logging_obj=logging_obj, + ), + model=model, + custom_llm_provider="bedrock", + logging_obj=logging_obj, + ) + + return streaming_response + ### COMPLETION + + if client is None or isinstance(client, AsyncHTTPHandler): + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + client = HTTPHandler(**_params) # type: ignore + else: + client = client + try: + response = client.post(url=prepped.url, headers=prepped.headers, data=data) # type: ignore + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=response.text) + except httpx.TimeoutException: + raise BedrockError(status_code=408, message="Timeout error occurred.") + + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + optional_params=optional_params, + api_key="", + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + ) def get_response_stream_shape(): @@ -1024,6 +1859,61 @@ class AWSEventStreamDecoder: self.model = model self.parser = EventStreamJSONParser() + def converse_chunk_parser(self, chunk_data: dict) -> GenericStreamingChunk: + try: + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + is_finished = False + finish_reason = "" + usage: Optional[ConverseTokenUsageBlock] = None + + index = int(chunk_data.get("contentBlockIndex", 0)) + if "start" in chunk_data: + start_obj = ContentBlockStartEvent(**chunk_data["start"]) + if ( + start_obj is not None + and "toolUse" in start_obj + and start_obj["toolUse"] is not None + ): + tool_use = { + "id": start_obj["toolUse"]["toolUseId"], + "type": "function", + "function": { + "name": start_obj["toolUse"]["name"], + "arguments": "", + }, + } + elif "delta" in chunk_data: + delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"]) + if "text" in delta_obj: + text = delta_obj["text"] + elif "toolUse" in delta_obj: + tool_use = { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": delta_obj["toolUse"]["input"], + }, + } + elif "stopReason" in chunk_data: + finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop")) + is_finished = True + elif "usage" in chunk_data: + usage = ConverseTokenUsageBlock(**chunk_data["usage"]) # type: ignore + + response = GenericStreamingChunk( + text=text, + tool_use=tool_use, + is_finished=is_finished, + finish_reason=finish_reason, + usage=usage, + index=index, + ) + return response + except Exception as e: + raise Exception("Received streaming error - {}".format(str(e))) + def _chunk_parser(self, chunk_data: dict) -> GenericStreamingChunk: text = "" is_finished = False @@ -1031,24 +1921,17 @@ class AWSEventStreamDecoder: if "outputText" in chunk_data: text = chunk_data["outputText"] # ai21 mapping - if "ai21" in self.model: # fake ai21 streaming + elif "ai21" in self.model: # fake ai21 streaming text = chunk_data.get("completions")[0].get("data").get("text") # type: ignore is_finished = True finish_reason = "stop" ######## bedrock.anthropic mappings ############### - elif "completion" in chunk_data: # not claude-3 - text = chunk_data["completion"] # bedrock.anthropic - stop_reason = chunk_data.get("stop_reason", None) - if stop_reason != None: - is_finished = True - finish_reason = stop_reason - elif "delta" in chunk_data: - if chunk_data["delta"].get("text", None) is not None: - text = chunk_data["delta"]["text"] - stop_reason = chunk_data["delta"].get("stop_reason", None) - if stop_reason != None: - is_finished = True - finish_reason = stop_reason + elif ( + "contentBlockIndex" in chunk_data + or "stopReason" in chunk_data + or "metrics" in chunk_data + ): + return self.converse_chunk_parser(chunk_data=chunk_data) ######## bedrock.mistral mappings ############### elif "outputs" in chunk_data: if ( @@ -1057,7 +1940,7 @@ class AWSEventStreamDecoder: ): text = chunk_data["outputs"][0]["text"] stop_reason = chunk_data.get("stop_reason", None) - if stop_reason != None: + if stop_reason is not None: is_finished = True finish_reason = stop_reason ######## bedrock.cohere mappings ############### @@ -1075,11 +1958,12 @@ class AWSEventStreamDecoder: is_finished = True finish_reason = chunk_data["completionReason"] return GenericStreamingChunk( - **{ - "text": text, - "is_finished": is_finished, - "finish_reason": finish_reason, - } + text=text, + is_finished=is_finished, + finish_reason=finish_reason, + usage=None, + index=0, + tool_use=None, ) def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[GenericStreamingChunk]: @@ -1116,9 +2000,14 @@ class AWSEventStreamDecoder: parsed_response = self.parser.parse(response_dict, get_response_stream_shape()) if response_dict["status_code"] != 200: raise ValueError(f"Bad response code, expected 200: {response_dict}") + if "chunk" in parsed_response: + chunk = parsed_response.get("chunk") + if not chunk: + return None + return chunk.get("bytes").decode() # type: ignore[no-any-return] + else: + chunk = response_dict.get("body") + if not chunk: + return None - chunk = parsed_response.get("chunk") - if not chunk: - return None - - return chunk.get("bytes").decode() # type: ignore[no-any-return] + return chunk.decode() # type: ignore[no-any-return] diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 4df25944b..5ec9c79bb 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1,4 +1,5 @@ -import httpx, asyncio +import litellm +import httpx, asyncio, traceback, os from typing import Optional, Union, Mapping, Any # https://www.python-httpx.org/advanced/timeouts @@ -11,6 +12,30 @@ class AsyncHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, concurrent_limit=1000, ): + async_proxy_mounts = None + # Check if the HTTP_PROXY and HTTPS_PROXY environment variables are set and use them accordingly. + http_proxy = os.getenv("HTTP_PROXY", None) + https_proxy = os.getenv("HTTPS_PROXY", None) + no_proxy = os.getenv("NO_PROXY", None) + ssl_verify = bool(os.getenv("SSL_VERIFY", litellm.ssl_verify)) + cert = os.getenv( + "SSL_CERTIFICATE", litellm.ssl_certificate + ) # /path/to/client.pem + + if http_proxy is not None and https_proxy is not None: + async_proxy_mounts = { + "http://": httpx.AsyncHTTPTransport(proxy=httpx.Proxy(url=http_proxy)), + "https://": httpx.AsyncHTTPTransport( + proxy=httpx.Proxy(url=https_proxy) + ), + } + # assume no_proxy is a list of comma separated urls + if no_proxy is not None and isinstance(no_proxy, str): + no_proxy_urls = no_proxy.split(",") + + for url in no_proxy_urls: # set no-proxy support for specific urls + async_proxy_mounts[url] = None # type: ignore + if timeout is None: timeout = _DEFAULT_TIMEOUT # Create a client with a connection pool @@ -20,6 +45,9 @@ class AsyncHTTPHandler: max_connections=concurrent_limit, max_keepalive_connections=concurrent_limit, ), + verify=ssl_verify, + mounts=async_proxy_mounts, + cert=cert, ) async def close(self): @@ -43,15 +71,22 @@ class AsyncHTTPHandler: self, url: str, data: Optional[Union[dict, str]] = None, # type: ignore + json: Optional[dict] = None, params: Optional[dict] = None, headers: Optional[dict] = None, stream: bool = False, ): - req = self.client.build_request( - "POST", url, data=data, params=params, headers=headers # type: ignore - ) - response = await self.client.send(req, stream=stream) - return response + try: + req = self.client.build_request( + "POST", url, data=data, json=json, params=params, headers=headers # type: ignore + ) + response = await self.client.send(req, stream=stream) + response.raise_for_status() + return response + except httpx.HTTPStatusError as e: + raise e + except Exception as e: + raise e def __del__(self) -> None: try: @@ -70,6 +105,28 @@ class HTTPHandler: if timeout is None: timeout = _DEFAULT_TIMEOUT + # Check if the HTTP_PROXY and HTTPS_PROXY environment variables are set and use them accordingly. + http_proxy = os.getenv("HTTP_PROXY", None) + https_proxy = os.getenv("HTTPS_PROXY", None) + no_proxy = os.getenv("NO_PROXY", None) + ssl_verify = bool(os.getenv("SSL_VERIFY", litellm.ssl_verify)) + cert = os.getenv( + "SSL_CERTIFICATE", litellm.ssl_certificate + ) # /path/to/client.pem + + sync_proxy_mounts = None + if http_proxy is not None and https_proxy is not None: + sync_proxy_mounts = { + "http://": httpx.HTTPTransport(proxy=httpx.Proxy(url=http_proxy)), + "https://": httpx.HTTPTransport(proxy=httpx.Proxy(url=https_proxy)), + } + # assume no_proxy is a list of comma separated urls + if no_proxy is not None and isinstance(no_proxy, str): + no_proxy_urls = no_proxy.split(",") + + for url in no_proxy_urls: # set no-proxy support for specific urls + sync_proxy_mounts[url] = None # type: ignore + if client is None: # Create a client with a connection pool self.client = httpx.Client( @@ -78,6 +135,9 @@ class HTTPHandler: max_connections=concurrent_limit, max_keepalive_connections=concurrent_limit, ), + verify=ssl_verify, + mounts=sync_proxy_mounts, + cert=cert, ) else: self.client = client @@ -96,12 +156,13 @@ class HTTPHandler: self, url: str, data: Optional[Union[dict, str]] = None, + json: Optional[Union[dict, str]] = None, params: Optional[dict] = None, headers: Optional[dict] = None, stream: bool = False, ): req = self.client.build_request( - "POST", url, data=data, params=params, headers=headers # type: ignore + "POST", url, data=data, json=json, params=params, headers=headers # type: ignore ) response = self.client.send(req, stream=stream) return response diff --git a/litellm/llms/databricks.py b/litellm/llms/databricks.py index 7b2013710..4fe475259 100644 --- a/litellm/llms/databricks.py +++ b/litellm/llms/databricks.py @@ -1,5 +1,6 @@ # What is this? ## Handler file for databricks API https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#chat-request +from functools import partial import os, types import json from enum import Enum @@ -123,7 +124,7 @@ class DatabricksConfig: original_chunk = None # this is used for function/tool calling chunk_data = chunk_data.replace("data:", "") chunk_data = chunk_data.strip() - if len(chunk_data) == 0: + if len(chunk_data) == 0 or chunk_data == "[DONE]": return { "text": "", "is_finished": is_finished, @@ -221,6 +222,32 @@ class DatabricksEmbeddingConfig: return optional_params +async def make_call( + client: AsyncHTTPHandler, + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj, +): + response = await client.post(api_base, headers=headers, data=data, stream=True) + + if response.status_code != 200: + raise DatabricksError(status_code=response.status_code, message=response.text) + + completion_stream = response.aiter_lines() + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response=completion_stream, # Pass the completion stream for logging + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + + class DatabricksChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() @@ -354,29 +381,21 @@ class DatabricksChatCompletion(BaseLLM): litellm_params=None, logger_fn=None, headers={}, - ): - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) - ) + client: Optional[AsyncHTTPHandler] = None, + ) -> CustomStreamWrapper: + data["stream"] = True - try: - response = await self.async_handler.post( - api_base, headers=headers, data=json.dumps(data), stream=True - ) - response.raise_for_status() - - completion_stream = response.aiter_lines() - except httpx.HTTPStatusError as e: - raise DatabricksError( - status_code=e.response.status_code, message=response.text - ) - except httpx.TimeoutException as e: - raise DatabricksError(status_code=408, message="Timeout error occurred.") - except Exception as e: - raise DatabricksError(status_code=500, message=str(e)) - streamwrapper = CustomStreamWrapper( - completion_stream=completion_stream, + completion_stream=None, + make_call=partial( + make_call, + api_base=api_base, + headers=headers, + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + ), model=model, custom_llm_provider="databricks", logging_obj=logging_obj, @@ -475,6 +494,8 @@ class DatabricksChatCompletion(BaseLLM): }, ) if acompletion == True: + if client is not None and isinstance(client, HTTPHandler): + client = None if ( stream is not None and stream == True ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) @@ -496,6 +517,7 @@ class DatabricksChatCompletion(BaseLLM): litellm_params=litellm_params, logger_fn=logger_fn, headers=headers, + client=client, ) else: return self.acompletion_function( diff --git a/litellm/llms/gemini.py b/litellm/llms/gemini.py index a55b39aef..cfdf39eca 100644 --- a/litellm/llms/gemini.py +++ b/litellm/llms/gemini.py @@ -1,13 +1,14 @@ -import os, types, traceback, copy, asyncio -import json -from enum import Enum +import types +import traceback +import copy import time from typing import Callable, Optional -from litellm.utils import ModelResponse, get_secret, Choices, Message, Usage +from litellm.utils import ModelResponse, Choices, Message, Usage import litellm -import sys, httpx +import httpx from .prompt_templates.factory import prompt_factory, custom_prompt, get_system_prompt from packaging.version import Version +from litellm import verbose_logger class GeminiError(Exception): @@ -264,7 +265,8 @@ def completion( choices_list.append(choice_obj) model_response["choices"] = choices_list except Exception as e: - traceback.print_exc() + verbose_logger.error("LiteLLM.gemini.py: Exception occured - {}".format(str(e))) + verbose_logger.debug(traceback.format_exc()) raise GeminiError( message=traceback.format_exc(), status_code=response.status_code ) @@ -356,7 +358,8 @@ async def async_completion( choices_list.append(choice_obj) model_response["choices"] = choices_list except Exception as e: - traceback.print_exc() + verbose_logger.error("LiteLLM.gemini.py: Exception occured - {}".format(str(e))) + verbose_logger.debug(traceback.format_exc()) raise GeminiError( message=traceback.format_exc(), status_code=response.status_code ) diff --git a/litellm/llms/ollama.py b/litellm/llms/ollama.py index 9c9b5e898..e7dd1d5f5 100644 --- a/litellm/llms/ollama.py +++ b/litellm/llms/ollama.py @@ -2,10 +2,12 @@ from itertools import chain import requests, types, time # type: ignore import json, uuid import traceback -from typing import Optional +from typing import Optional, List import litellm +from litellm.types.utils import ProviderField import httpx, aiohttp, asyncio # type: ignore from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm import verbose_logger class OllamaError(Exception): @@ -45,6 +47,8 @@ class OllamaConfig: - `temperature` (float): The temperature of the model. Increasing the temperature will make the model answer more creatively. Default: 0.8. Example usage: temperature 0.7 + - `seed` (int): Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. Example usage: seed 42 + - `stop` (string[]): Sets the stop sequences to use. Example usage: stop "AI assistant:" - `tfs_z` (float): Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. Default: 1. Example usage: tfs_z 1 @@ -69,6 +73,7 @@ class OllamaConfig: repeat_last_n: Optional[int] = None repeat_penalty: Optional[float] = None temperature: Optional[float] = None + seed: Optional[int] = None stop: Optional[list] = ( None # stop is a list based on this - https://github.com/ollama/ollama/pull/442 ) @@ -90,6 +95,7 @@ class OllamaConfig: repeat_last_n: Optional[int] = None, repeat_penalty: Optional[float] = None, temperature: Optional[float] = None, + seed: Optional[int] = None, stop: Optional[list] = None, tfs_z: Optional[float] = None, num_predict: Optional[int] = None, @@ -120,6 +126,59 @@ class OllamaConfig: ) and v is not None } + + def get_required_params(self) -> List[ProviderField]: + """For a given provider, return it's required fields with a description""" + return [ + ProviderField( + field_name="base_url", + field_type="string", + field_description="Your Ollama API Base", + field_value="http://10.10.11.249:11434", + ) + ] + + + def get_supported_openai_params( + self, + ): + return [ + "max_tokens", + "stream", + "top_p", + "temperature", + "seed", + "frequency_penalty", + "stop", + "response_format", + ] + + +# ollama wants plain base64 jpeg/png files as images. strip any leading dataURI +# and convert to jpeg if necessary. +def _convert_image(image): + import base64, io + + try: + from PIL import Image + except: + raise Exception( + "ollama image conversion failed please run `pip install Pillow`" + ) + + orig = image + if image.startswith("data:"): + image = image.split(",")[-1] + try: + image_data = Image.open(io.BytesIO(base64.b64decode(image))) + if image_data.format in ["JPEG", "PNG"]: + return image + except: + return orig + jpeg_image = io.BytesIO() + image_data.convert("RGB").save(jpeg_image, "JPEG") + jpeg_image.seek(0) + return base64.b64encode(jpeg_image.getvalue()).decode("utf-8") # ollama implementation @@ -158,7 +217,7 @@ def get_ollama_response( if format is not None: data["format"] = format if images is not None: - data["images"] = images + data["images"] = [_convert_image(image) for image in images] ## LOGGING logging_obj.pre_call( @@ -349,7 +408,13 @@ async def ollama_async_streaming(url, data, model_response, encoding, logging_ob async for transformed_chunk in streamwrapper: yield transformed_chunk except Exception as e: - traceback.print_exc() + verbose_logger.error( + "LiteLLM.ollama.py::ollama_async_streaming(): Exception occured - {}".format( + str(e) + ) + ) + verbose_logger.debug(traceback.format_exc()) + raise e @@ -413,7 +478,12 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj): ) return model_response except Exception as e: - traceback.print_exc() + verbose_logger.error( + "LiteLLM.ollama.py::ollama_acompletion(): Exception occured - {}".format( + str(e) + ) + ) + verbose_logger.debug(traceback.format_exc()) raise e diff --git a/litellm/llms/ollama_chat.py b/litellm/llms/ollama_chat.py index d1ff4953f..a7439bbcc 100644 --- a/litellm/llms/ollama_chat.py +++ b/litellm/llms/ollama_chat.py @@ -1,11 +1,15 @@ from itertools import chain -import requests, types, time -import json, uuid +import requests +import types +import time +import json +import uuid import traceback from typing import Optional +from litellm import verbose_logger import litellm -import httpx, aiohttp, asyncio -from .prompt_templates.factory import prompt_factory, custom_prompt +import httpx +import aiohttp class OllamaError(Exception): @@ -45,6 +49,8 @@ class OllamaChatConfig: - `temperature` (float): The temperature of the model. Increasing the temperature will make the model answer more creatively. Default: 0.8. Example usage: temperature 0.7 + - `seed` (int): Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. Example usage: seed 42 + - `stop` (string[]): Sets the stop sequences to use. Example usage: stop "AI assistant:" - `tfs_z` (float): Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. Default: 1. Example usage: tfs_z 1 @@ -69,6 +75,7 @@ class OllamaChatConfig: repeat_last_n: Optional[int] = None repeat_penalty: Optional[float] = None temperature: Optional[float] = None + seed: Optional[int] = None stop: Optional[list] = ( None # stop is a list based on this - https://github.com/ollama/ollama/pull/442 ) @@ -90,6 +97,7 @@ class OllamaChatConfig: repeat_last_n: Optional[int] = None, repeat_penalty: Optional[float] = None, temperature: Optional[float] = None, + seed: Optional[int] = None, stop: Optional[list] = None, tfs_z: Optional[float] = None, num_predict: Optional[int] = None, @@ -130,6 +138,7 @@ class OllamaChatConfig: "stream", "top_p", "temperature", + "seed", "frequency_penalty", "stop", "tools", @@ -146,6 +155,8 @@ class OllamaChatConfig: optional_params["stream"] = value if param == "temperature": optional_params["temperature"] = value + if param == "seed": + optional_params["seed"] = value if param == "top_p": optional_params["top_p"] = value if param == "frequency_penalty": @@ -292,7 +303,10 @@ def get_ollama_response( tool_calls=[ { "id": f"call_{str(uuid.uuid4())}", - "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, "type": "function", } ], @@ -300,7 +314,9 @@ def get_ollama_response( model_response["choices"][0]["message"] = message model_response["choices"][0]["finish_reason"] = "tool_calls" else: - model_response["choices"][0]["message"]["content"] = response_json["message"]["content"] + model_response["choices"][0]["message"]["content"] = response_json["message"][ + "content" + ] model_response["created"] = int(time.time()) model_response["model"] = "ollama/" + model prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore @@ -354,7 +370,10 @@ def ollama_completion_stream(url, api_key, data, logging_obj): tool_calls=[ { "id": f"call_{str(uuid.uuid4())}", - "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, "type": "function", } ], @@ -403,9 +422,10 @@ async def ollama_async_streaming( first_chunk_content = first_chunk.choices[0].delta.content or "" response_content = first_chunk_content + "".join( [ - chunk.choices[0].delta.content - async for chunk in streamwrapper - if chunk.choices[0].delta.content] + chunk.choices[0].delta.content + async for chunk in streamwrapper + if chunk.choices[0].delta.content + ] ) function_call = json.loads(response_content) delta = litellm.utils.Delta( @@ -413,7 +433,10 @@ async def ollama_async_streaming( tool_calls=[ { "id": f"call_{str(uuid.uuid4())}", - "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, "type": "function", } ], @@ -426,7 +449,8 @@ async def ollama_async_streaming( async for transformed_chunk in streamwrapper: yield transformed_chunk except Exception as e: - traceback.print_exc() + verbose_logger.error("LiteLLM.gemini(): Exception occured - {}".format(str(e))) + verbose_logger.debug(traceback.format_exc()) async def ollama_acompletion( @@ -476,7 +500,10 @@ async def ollama_acompletion( tool_calls=[ { "id": f"call_{str(uuid.uuid4())}", - "function": {"name": function_call["name"], "arguments": json.dumps(function_call["arguments"])}, + "function": { + "name": function_call["name"], + "arguments": json.dumps(function_call["arguments"]), + }, "type": "function", } ], @@ -484,7 +511,9 @@ async def ollama_acompletion( model_response["choices"][0]["message"] = message model_response["choices"][0]["finish_reason"] = "tool_calls" else: - model_response["choices"][0]["message"]["content"] = response_json["message"]["content"] + model_response["choices"][0]["message"]["content"] = response_json[ + "message" + ]["content"] model_response["created"] = int(time.time()) model_response["model"] = "ollama_chat/" + data["model"] @@ -502,5 +531,9 @@ async def ollama_acompletion( ) return model_response except Exception as e: - traceback.print_exc() + verbose_logger.error( + "LiteLLM.ollama_acompletion(): Exception occured - {}".format(str(e)) + ) + verbose_logger.debug(traceback.format_exc()) + raise e diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 6197ec922..dec86d35d 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -6,7 +6,8 @@ from typing import ( Literal, Iterable, ) -from typing_extensions import override +import hashlib +from typing_extensions import override, overload from pydantic import BaseModel import types, time, json, traceback import httpx @@ -21,11 +22,12 @@ from litellm.utils import ( TranscriptionResponse, TextCompletionResponse, ) -from typing import Callable, Optional +from typing import Callable, Optional, Coroutine import litellm from .prompt_templates.factory import prompt_factory, custom_prompt from openai import OpenAI, AsyncOpenAI from ..types.llms.openai import * +import openai class OpenAIError(Exception): @@ -224,6 +226,7 @@ class DeepInfraConfig: def get_supported_openai_params(self): return [ + "stream", "frequency_penalty", "function_call", "functions", @@ -348,7 +351,6 @@ class OpenAIConfig: "top_p", "tools", "tool_choice", - "user", "function_call", "functions", "max_retries", @@ -361,6 +363,12 @@ class OpenAIConfig: ): # gpt-4 does not support 'response_format' model_specific_params.append("response_format") + if ( + model in litellm.open_ai_chat_completion_models + ) or model in litellm.open_ai_text_completion_models: + model_specific_params.append( + "user" + ) # user is not a param supported by all openai-compatible endpoints - e.g. azure ai return base_params + model_specific_params def map_openai_params( @@ -497,6 +505,64 @@ class OpenAIChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() + def _get_openai_client( + self, + is_async: bool, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + timeout: Union[float, httpx.Timeout] = httpx.Timeout(None), + max_retries: Optional[int] = None, + organization: Optional[str] = None, + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + ): + args = locals() + if client is None: + if not isinstance(max_retries, int): + raise OpenAIError( + status_code=422, + message="max retries must be an int. Passed in value: {}".format( + max_retries + ), + ) + # Creating a new OpenAI Client + # check in memory cache before creating a new one + # Convert the API key to bytes + hashed_api_key = None + if api_key is not None: + hash_object = hashlib.sha256(api_key.encode()) + # Hexadecimal representation of the hash + hashed_api_key = hash_object.hexdigest() + + _cache_key = f"hashed_api_key={hashed_api_key},api_base={api_base},timeout={timeout},max_retries={max_retries},organization={organization},is_async={is_async}" + + if _cache_key in litellm.in_memory_llm_clients_cache: + return litellm.in_memory_llm_clients_cache[_cache_key] + if is_async: + _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI( + api_key=api_key, + base_url=api_base, + http_client=litellm.aclient_session, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) + else: + _new_client = OpenAI( + api_key=api_key, + base_url=api_base, + http_client=litellm.client_session, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) + + ## SAVE CACHE KEY + litellm.in_memory_llm_clients_cache[_cache_key] = _new_client + return _new_client + + else: + return client + def completion( self, model_response: ModelResponse, @@ -603,17 +669,16 @@ class OpenAIChatCompletion(BaseLLM): raise OpenAIError( status_code=422, message="max retries must be an int" ) - if client is None: - openai_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.client_session, - timeout=timeout, - max_retries=max_retries, - organization=organization, - ) - else: - openai_client = client + + openai_client = self._get_openai_client( + is_async=False, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) ## LOGGING logging_obj.pre_call( @@ -693,17 +758,15 @@ class OpenAIChatCompletion(BaseLLM): ): response = None try: - if client is None: - openai_aclient = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.aclient_session, - timeout=timeout, - max_retries=max_retries, - organization=organization, - ) - else: - openai_aclient = client + openai_aclient = self._get_openai_client( + is_async=True, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) ## LOGGING logging_obj.pre_call( @@ -747,17 +810,15 @@ class OpenAIChatCompletion(BaseLLM): max_retries=None, headers=None, ): - if client is None: - openai_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.client_session, - timeout=timeout, - max_retries=max_retries, - organization=organization, - ) - else: - openai_client = client + openai_client = self._get_openai_client( + is_async=False, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) ## LOGGING logging_obj.pre_call( input=data["messages"], @@ -794,17 +855,15 @@ class OpenAIChatCompletion(BaseLLM): ): response = None try: - if client is None: - openai_aclient = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.aclient_session, - timeout=timeout, - max_retries=max_retries, - organization=organization, - ) - else: - openai_aclient = client + openai_aclient = self._get_openai_client( + is_async=True, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) ## LOGGING logging_obj.pre_call( input=data["messages"], @@ -858,16 +917,14 @@ class OpenAIChatCompletion(BaseLLM): ): response = None try: - if client is None: - openai_aclient = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.aclient_session, - timeout=timeout, - max_retries=max_retries, - ) - else: - openai_aclient = client + openai_aclient = self._get_openai_client( + is_async=True, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) response = await openai_aclient.embeddings.create(**data, timeout=timeout) # type: ignore stringified_response = response.model_dump() ## LOGGING @@ -915,19 +972,18 @@ class OpenAIChatCompletion(BaseLLM): additional_args={"complete_input_dict": data, "api_base": api_base}, ) - if aembedding == True: + if aembedding is True: response = self.aembedding(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries) # type: ignore return response - if client is None: - openai_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.client_session, - timeout=timeout, - max_retries=max_retries, - ) - else: - openai_client = client + + openai_client = self._get_openai_client( + is_async=False, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) ## COMPLETION CALL response = openai_client.embeddings.create(**data, timeout=timeout) # type: ignore @@ -963,16 +1019,16 @@ class OpenAIChatCompletion(BaseLLM): ): response = None try: - if client is None: - openai_aclient = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.aclient_session, - timeout=timeout, - max_retries=max_retries, - ) - else: - openai_aclient = client + + openai_aclient = self._get_openai_client( + is_async=True, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + response = await openai_aclient.images.generate(**data, timeout=timeout) # type: ignore stringified_response = response.model_dump() ## LOGGING @@ -1017,16 +1073,14 @@ class OpenAIChatCompletion(BaseLLM): response = self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries) # type: ignore return response - if client is None: - openai_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.client_session, - timeout=timeout, - max_retries=max_retries, - ) - else: - openai_client = client + openai_client = self._get_openai_client( + is_async=False, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) ## LOGGING logging_obj.pre_call( @@ -1084,14 +1138,14 @@ class OpenAIChatCompletion(BaseLLM): model_response: TranscriptionResponse, timeout: float, max_retries: int, - api_key: Optional[str] = None, - api_base: Optional[str] = None, + api_key: Optional[str], + api_base: Optional[str], client=None, logging_obj=None, atranscription: bool = False, ): data = {"model": model, "file": audio_file, **optional_params} - if atranscription == True: + if atranscription is True: return self.async_audio_transcriptions( audio_file=audio_file, data=data, @@ -1103,16 +1157,14 @@ class OpenAIChatCompletion(BaseLLM): max_retries=max_retries, logging_obj=logging_obj, ) - if client is None: - openai_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.client_session, - timeout=timeout, - max_retries=max_retries, - ) - else: - openai_client = client + + openai_client = self._get_openai_client( + is_async=False, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + ) response = openai_client.audio.transcriptions.create( **data, timeout=timeout # type: ignore ) @@ -1141,18 +1193,16 @@ class OpenAIChatCompletion(BaseLLM): max_retries=None, logging_obj=None, ): - response = None try: - if client is None: - openai_aclient = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=litellm.aclient_session, - timeout=timeout, - max_retries=max_retries, - ) - else: - openai_aclient = client + openai_aclient = self._get_openai_client( + is_async=True, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + response = await openai_aclient.audio.transcriptions.create( **data, timeout=timeout ) # type: ignore @@ -1175,6 +1225,87 @@ class OpenAIChatCompletion(BaseLLM): ) raise e + def audio_speech( + self, + model: str, + input: str, + voice: str, + optional_params: dict, + api_key: Optional[str], + api_base: Optional[str], + organization: Optional[str], + project: Optional[str], + max_retries: int, + timeout: Union[float, httpx.Timeout], + aspeech: Optional[bool] = None, + client=None, + ) -> HttpxBinaryResponseContent: + + if aspeech is not None and aspeech is True: + return self.async_audio_speech( + model=model, + input=input, + voice=voice, + optional_params=optional_params, + api_key=api_key, + api_base=api_base, + organization=organization, + project=project, + max_retries=max_retries, + timeout=timeout, + client=client, + ) # type: ignore + + openai_client = self._get_openai_client( + is_async=False, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = openai_client.audio.speech.create( + model=model, + voice=voice, # type: ignore + input=input, + **optional_params, + ) + return response + + async def async_audio_speech( + self, + model: str, + input: str, + voice: str, + optional_params: dict, + api_key: Optional[str], + api_base: Optional[str], + organization: Optional[str], + project: Optional[str], + max_retries: int, + timeout: Union[float, httpx.Timeout], + client=None, + ) -> HttpxBinaryResponseContent: + + openai_client = self._get_openai_client( + is_async=True, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + client=client, + ) + + response = await openai_client.audio.speech.create( + model=model, + voice=voice, # type: ignore + input=input, + **optional_params, + ) + + return response + async def ahealth_check( self, model: Optional[str], @@ -1496,6 +1627,322 @@ class OpenAITextCompletion(BaseLLM): yield transformed_chunk +class OpenAIFilesAPI(BaseLLM): + """ + OpenAI methods to support for batches + - create_file() + - retrieve_file() + - list_files() + - delete_file() + - file_content() + - update_file() + """ + + def __init__(self) -> None: + super().__init__() + + def get_openai_client( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + _is_async: bool = False, + ) -> Optional[Union[OpenAI, AsyncOpenAI]]: + received_args = locals() + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = None + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client" or k == "_is_async": + pass + elif k == "api_base" and v is not None: + data["base_url"] = v + elif v is not None: + data[k] = v + if _is_async is True: + openai_client = AsyncOpenAI(**data) + else: + openai_client = OpenAI(**data) # type: ignore + else: + openai_client = client + + return openai_client + + async def acreate_file( + self, + create_file_data: CreateFileRequest, + openai_client: AsyncOpenAI, + ) -> FileObject: + response = await openai_client.files.create(**create_file_data) + return response + + def create_file( + self, + _is_async: bool, + create_file_data: CreateFileRequest, + api_base: str, + api_key: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + ) -> Union[FileObject, Coroutine[Any, Any, FileObject]]: + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.acreate_file( # type: ignore + create_file_data=create_file_data, openai_client=openai_client + ) + response = openai_client.files.create(**create_file_data) + return response + + async def afile_content( + self, + file_content_request: FileContentRequest, + openai_client: AsyncOpenAI, + ) -> HttpxBinaryResponseContent: + response = await openai_client.files.content(**file_content_request) + return response + + def file_content( + self, + _is_async: bool, + file_content_request: FileContentRequest, + api_base: str, + api_key: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + ) -> Union[ + HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent] + ]: + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.afile_content( # type: ignore + file_content_request=file_content_request, + openai_client=openai_client, + ) + response = openai_client.files.content(**file_content_request) + + return response + + +class OpenAIBatchesAPI(BaseLLM): + """ + OpenAI methods to support for batches + - create_batch() + - retrieve_batch() + - cancel_batch() + - list_batch() + """ + + def __init__(self) -> None: + super().__init__() + + def get_openai_client( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + _is_async: bool = False, + ) -> Optional[Union[OpenAI, AsyncOpenAI]]: + received_args = locals() + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = None + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client" or k == "_is_async": + pass + elif k == "api_base" and v is not None: + data["base_url"] = v + elif v is not None: + data[k] = v + if _is_async is True: + openai_client = AsyncOpenAI(**data) + else: + openai_client = OpenAI(**data) # type: ignore + else: + openai_client = client + + return openai_client + + async def acreate_batch( + self, + create_batch_data: CreateBatchRequest, + openai_client: AsyncOpenAI, + ) -> Batch: + response = await openai_client.batches.create(**create_batch_data) + return response + + def create_batch( + self, + _is_async: bool, + create_batch_data: CreateBatchRequest, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + ) -> Union[Batch, Coroutine[Any, Any, Batch]]: + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.acreate_batch( # type: ignore + create_batch_data=create_batch_data, openai_client=openai_client + ) + response = openai_client.batches.create(**create_batch_data) + return response + + async def aretrieve_batch( + self, + retrieve_batch_data: RetrieveBatchRequest, + openai_client: AsyncOpenAI, + ) -> Batch: + response = await openai_client.batches.retrieve(**retrieve_batch_data) + return response + + def retrieve_batch( + self, + _is_async: bool, + retrieve_batch_data: RetrieveBatchRequest, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI] = None, + ): + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.aretrieve_batch( # type: ignore + retrieve_batch_data=retrieve_batch_data, openai_client=openai_client + ) + response = openai_client.batches.retrieve(**retrieve_batch_data) + return response + + def cancel_batch( + self, + _is_async: bool, + cancel_batch_data: CancelBatchRequest, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI] = None, + ): + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + response = openai_client.batches.cancel(**cancel_batch_data) + return response + + # def list_batch( + # self, + # list_batch_data: ListBatchRequest, + # api_key: Optional[str], + # api_base: Optional[str], + # timeout: Union[float, httpx.Timeout], + # max_retries: Optional[int], + # organization: Optional[str], + # client: Optional[OpenAI] = None, + # ): + # openai_client: OpenAI = self.get_openai_client( + # api_key=api_key, + # api_base=api_base, + # timeout=timeout, + # max_retries=max_retries, + # organization=organization, + # client=client, + # ) + # response = openai_client.batches.list(**list_batch_data) + # return response + + class OpenAIAssistantsAPI(BaseLLM): def __init__(self) -> None: super().__init__() @@ -1525,8 +1972,85 @@ class OpenAIAssistantsAPI(BaseLLM): return openai_client + def async_get_openai_client( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI] = None, + ) -> AsyncOpenAI: + received_args = locals() + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client": + pass + elif k == "api_base" and v is not None: + data["base_url"] = v + elif v is not None: + data[k] = v + openai_client = AsyncOpenAI(**data) # type: ignore + else: + openai_client = client + + return openai_client + ### ASSISTANTS ### + async def async_get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + ) -> AsyncCursorPage[Assistant]: + openai_client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = await openai_client.beta.assistants.list() + + return response + + # fmt: off + + @overload + def get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + aget_assistants: Literal[True], + ) -> Coroutine[None, None, AsyncCursorPage[Assistant]]: + ... + + @overload + def get_assistants( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + aget_assistants: Optional[Literal[False]], + ) -> SyncCursorPage[Assistant]: + ... + + # fmt: on + def get_assistants( self, api_key: Optional[str], @@ -1534,8 +2058,18 @@ class OpenAIAssistantsAPI(BaseLLM): timeout: Union[float, httpx.Timeout], max_retries: Optional[int], organization: Optional[str], - client: Optional[OpenAI], - ) -> SyncCursorPage[Assistant]: + client=None, + aget_assistants=None, + ): + if aget_assistants is not None and aget_assistants == True: + return self.async_get_assistants( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) openai_client = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -1551,18 +2085,95 @@ class OpenAIAssistantsAPI(BaseLLM): ### MESSAGES ### - def add_message( + async def a_add_message( self, thread_id: str, - message_data: MessageData, + message_data: dict, api_key: Optional[str], api_base: Optional[str], timeout: Union[float, httpx.Timeout], max_retries: Optional[int], organization: Optional[str], - client: Optional[OpenAI] = None, + client: Optional[AsyncOpenAI] = None, ) -> OpenAIMessage: + openai_client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + thread_message: OpenAIMessage = await openai_client.beta.threads.messages.create( # type: ignore + thread_id, **message_data # type: ignore + ) + + response_obj: Optional[OpenAIMessage] = None + if getattr(thread_message, "status", None) is None: + thread_message.status = "completed" + response_obj = OpenAIMessage(**thread_message.dict()) + else: + response_obj = OpenAIMessage(**thread_message.dict()) + return response_obj + + # fmt: off + + @overload + def add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + a_add_message: Literal[True], + ) -> Coroutine[None, None, OpenAIMessage]: + ... + + @overload + def add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + a_add_message: Optional[Literal[False]], + ) -> OpenAIMessage: + ... + + # fmt: on + + def add_message( + self, + thread_id: str, + message_data: dict, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client=None, + a_add_message: Optional[bool] = None, + ): + if a_add_message is not None and a_add_message == True: + return self.a_add_message( + thread_id=thread_id, + message_data=message_data, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) openai_client = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -1584,6 +2195,61 @@ class OpenAIAssistantsAPI(BaseLLM): response_obj = OpenAIMessage(**thread_message.dict()) return response_obj + async def async_get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI] = None, + ) -> AsyncCursorPage[OpenAIMessage]: + openai_client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = await openai_client.beta.threads.messages.list(thread_id=thread_id) + + return response + + # fmt: off + + @overload + def get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + aget_messages: Literal[True], + ) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]: + ... + + @overload + def get_messages( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + aget_messages: Optional[Literal[False]], + ) -> SyncCursorPage[OpenAIMessage]: + ... + + # fmt: on + def get_messages( self, thread_id: str, @@ -1592,8 +2258,19 @@ class OpenAIAssistantsAPI(BaseLLM): timeout: Union[float, httpx.Timeout], max_retries: Optional[int], organization: Optional[str], - client: Optional[OpenAI] = None, - ) -> SyncCursorPage[OpenAIMessage]: + client=None, + aget_messages=None, + ): + if aget_messages is not None and aget_messages == True: + return self.async_get_messages( + thread_id=thread_id, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) openai_client = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -1609,6 +2286,70 @@ class OpenAIAssistantsAPI(BaseLLM): ### THREADS ### + async def async_create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + ) -> Thread: + openai_client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + data = {} + if messages is not None: + data["messages"] = messages # type: ignore + if metadata is not None: + data["metadata"] = metadata # type: ignore + + message_thread = await openai_client.beta.threads.create(**data) # type: ignore + + return Thread(**message_thread.dict()) + + # fmt: off + + @overload + def create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + client: Optional[AsyncOpenAI], + acreate_thread: Literal[True], + ) -> Coroutine[None, None, Thread]: + ... + + @overload + def create_thread( + self, + metadata: Optional[dict], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], + client: Optional[OpenAI], + acreate_thread: Optional[Literal[False]], + ) -> Thread: + ... + + # fmt: on + def create_thread( self, metadata: Optional[dict], @@ -1617,9 +2358,10 @@ class OpenAIAssistantsAPI(BaseLLM): timeout: Union[float, httpx.Timeout], max_retries: Optional[int], organization: Optional[str], - client: Optional[OpenAI], messages: Optional[Iterable[OpenAICreateThreadParamsMessage]], - ) -> Thread: + client=None, + acreate_thread=None, + ): """ Here's an example: ``` @@ -1630,6 +2372,17 @@ class OpenAIAssistantsAPI(BaseLLM): openai_api.create_thread(messages=[message]) ``` """ + if acreate_thread is not None and acreate_thread == True: + return self.async_create_thread( + metadata=metadata, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + messages=messages, + ) openai_client = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -1649,6 +2402,61 @@ class OpenAIAssistantsAPI(BaseLLM): return Thread(**message_thread.dict()) + async def async_get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + ) -> Thread: + openai_client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = await openai_client.beta.threads.retrieve(thread_id=thread_id) + + return Thread(**response.dict()) + + # fmt: off + + @overload + def get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + aget_thread: Literal[True], + ) -> Coroutine[None, None, Thread]: + ... + + @overload + def get_thread( + self, + thread_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[OpenAI], + aget_thread: Optional[Literal[False]], + ) -> Thread: + ... + + # fmt: on + def get_thread( self, thread_id: str, @@ -1657,8 +2465,19 @@ class OpenAIAssistantsAPI(BaseLLM): timeout: Union[float, httpx.Timeout], max_retries: Optional[int], organization: Optional[str], - client: Optional[OpenAI], - ) -> Thread: + client=None, + aget_thread=None, + ): + if aget_thread is not None and aget_thread == True: + return self.async_get_thread( + thread_id=thread_id, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) openai_client = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -1677,6 +2496,142 @@ class OpenAIAssistantsAPI(BaseLLM): ### RUNS ### + async def arun_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[AsyncOpenAI], + ) -> Run: + openai_client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + + response = await openai_client.beta.threads.runs.create_and_poll( # type: ignore + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + ) + + return response + + def async_run_thread_stream( + self, + client: AsyncOpenAI, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + tools: Optional[Iterable[AssistantToolParam]], + event_handler: Optional[AssistantEventHandler], + ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: + data = { + "thread_id": thread_id, + "assistant_id": assistant_id, + "additional_instructions": additional_instructions, + "instructions": instructions, + "metadata": metadata, + "model": model, + "tools": tools, + } + if event_handler is not None: + data["event_handler"] = event_handler + return client.beta.threads.runs.stream(**data) # type: ignore + + def run_thread_stream( + self, + client: OpenAI, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + tools: Optional[Iterable[AssistantToolParam]], + event_handler: Optional[AssistantEventHandler], + ) -> AssistantStreamManager[AssistantEventHandler]: + data = { + "thread_id": thread_id, + "assistant_id": assistant_id, + "additional_instructions": additional_instructions, + "instructions": instructions, + "metadata": metadata, + "model": model, + "tools": tools, + } + if event_handler is not None: + data["event_handler"] = event_handler + return client.beta.threads.runs.stream(**data) # type: ignore + + # fmt: off + + @overload + def run_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client, + arun_thread: Literal[True], + event_handler: Optional[AssistantEventHandler], + ) -> Coroutine[None, None, Run]: + ... + + @overload + def run_thread( + self, + thread_id: str, + assistant_id: str, + additional_instructions: Optional[str], + instructions: Optional[str], + metadata: Optional[object], + model: Optional[str], + stream: Optional[bool], + tools: Optional[Iterable[AssistantToolParam]], + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client, + arun_thread: Optional[Literal[False]], + event_handler: Optional[AssistantEventHandler], + ) -> Run: + ... + + # fmt: on + def run_thread( self, thread_id: str, @@ -1692,8 +2647,47 @@ class OpenAIAssistantsAPI(BaseLLM): timeout: Union[float, httpx.Timeout], max_retries: Optional[int], organization: Optional[str], - client: Optional[OpenAI], - ) -> Run: + client=None, + arun_thread=None, + event_handler: Optional[AssistantEventHandler] = None, + ): + if arun_thread is not None and arun_thread == True: + if stream is not None and stream == True: + _client = self.async_get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) + return self.async_run_thread_stream( + client=_client, + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + event_handler=event_handler, + ) + return self.arun_thread( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + stream=stream, + tools=tools, + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + ) openai_client = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -1703,6 +2697,19 @@ class OpenAIAssistantsAPI(BaseLLM): client=client, ) + if stream is not None and stream == True: + return self.run_thread_stream( + client=openai_client, + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + event_handler=event_handler, + ) + response = openai_client.beta.threads.runs.create_and_poll( # type: ignore thread_id=thread_id, assistant_id=assistant_id, diff --git a/litellm/llms/palm.py b/litellm/llms/palm.py index f15be43db..4d9953e77 100644 --- a/litellm/llms/palm.py +++ b/litellm/llms/palm.py @@ -1,11 +1,12 @@ -import os, types, traceback, copy -import json -from enum import Enum +import types +import traceback +import copy import time from typing import Callable, Optional -from litellm.utils import ModelResponse, get_secret, Choices, Message, Usage +from litellm.utils import ModelResponse, Choices, Message, Usage import litellm -import sys, httpx +import httpx +from litellm import verbose_logger class PalmError(Exception): @@ -165,7 +166,10 @@ def completion( choices_list.append(choice_obj) model_response["choices"] = choices_list except Exception as e: - traceback.print_exc() + verbose_logger.error( + "litellm.llms.palm.py::completion(): Exception occured - {}".format(str(e)) + ) + verbose_logger.debug(traceback.format_exc()) raise PalmError( message=traceback.format_exc(), status_code=response.status_code ) diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index 1e7e1d334..66c28acee 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -1,8 +1,9 @@ # What is this? ## Controller file for Predibase Integration - https://predibase.com/ - +from functools import partial import os, types +import traceback import json from enum import Enum import requests, copy # type: ignore @@ -51,6 +52,32 @@ class PredibaseError(Exception): ) # Call the base class constructor with the parameters it needs +async def make_call( + client: AsyncHTTPHandler, + api_base: str, + headers: dict, + data: str, + model: str, + messages: list, + logging_obj, +): + response = await client.post(api_base, headers=headers, data=data, stream=True) + + if response.status_code != 200: + raise PredibaseError(status_code=response.status_code, message=response.text) + + completion_stream = response.aiter_lines() + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response=completion_stream, # Pass the completion stream for logging + additional_args={"complete_input_dict": data}, + ) + + return completion_stream + + class PredibaseConfig: """ Reference: https://docs.predibase.com/user-guide/inference/rest_api @@ -126,11 +153,17 @@ class PredibaseChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() - def _validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: + def _validate_environment( + self, api_key: Optional[str], user_headers: dict, tenant_id: Optional[str] + ) -> dict: if api_key is None: raise ValueError( "Missing Predibase API Key - A call is being made to predibase but no key is set either in the environment variables or via params" ) + if tenant_id is None: + raise ValueError( + "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." + ) headers = { "content-type": "application/json", "Authorization": "Bearer {}".format(api_key), @@ -210,12 +243,12 @@ class PredibaseChatCompletion(BaseLLM): "details" in completion_response and "tokens" in completion_response["details"] ): - model_response.choices[0].finish_reason = completion_response[ - "details" - ]["finish_reason"] + model_response.choices[0].finish_reason = map_finish_reason( + completion_response["details"]["finish_reason"] + ) sum_logprob = 0 for token in completion_response["details"]["tokens"]: - if token["logprob"] != None: + if token["logprob"] is not None: sum_logprob += token["logprob"] model_response["choices"][0][ "message" @@ -233,7 +266,7 @@ class PredibaseChatCompletion(BaseLLM): ): sum_logprob = 0 for token in item["tokens"]: - if token["logprob"] != None: + if token["logprob"] is not None: sum_logprob += token["logprob"] if len(item["generated_text"]) > 0: message_obj = Message( @@ -243,7 +276,7 @@ class PredibaseChatCompletion(BaseLLM): else: message_obj = Message(content=None) choice_obj = Choices( - finish_reason=item["finish_reason"], + finish_reason=map_finish_reason(item["finish_reason"]), index=idx + 1, message=message_obj, ) @@ -253,10 +286,8 @@ class PredibaseChatCompletion(BaseLLM): ## CALCULATING USAGE prompt_tokens = 0 try: - prompt_tokens = len( - encoding.encode(model_response["choices"][0]["message"]["content"]) - ) ##[TODO] use a model-specific tokenizer here - except: + prompt_tokens = litellm.token_counter(messages=messages) + except Exception: # this should remain non blocking we should not block a response returning if calculating usage fails pass output_text = model_response["choices"][0]["message"].get("content", "") @@ -299,15 +330,17 @@ class PredibaseChatCompletion(BaseLLM): logging_obj, optional_params: dict, tenant_id: str, + timeout: Union[float, httpx.Timeout], acompletion=None, litellm_params=None, logger_fn=None, headers: dict = {}, ) -> Union[ModelResponse, CustomStreamWrapper]: - headers = self._validate_environment(api_key, headers) + headers = self._validate_environment(api_key, headers, tenant_id=tenant_id) completion_url = "" input_text = "" base_url = "https://serving.app.predibase.com" + if "https" in model: completion_url = model elif api_base: @@ -317,7 +350,7 @@ class PredibaseChatCompletion(BaseLLM): completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}" - if optional_params.get("stream", False) == True: + if optional_params.get("stream", False) is True: completion_url += "/generate_stream" else: completion_url += "/generate" @@ -361,9 +394,9 @@ class PredibaseChatCompletion(BaseLLM): }, ) ## COMPLETION CALL - if acompletion == True: + if acompletion is True: ### ASYNC STREAMING - if stream == True: + if stream is True: return self.async_streaming( model=model, messages=messages, @@ -378,6 +411,7 @@ class PredibaseChatCompletion(BaseLLM): litellm_params=litellm_params, logger_fn=logger_fn, headers=headers, + timeout=timeout, ) # type: ignore else: ### ASYNC COMPLETION @@ -396,10 +430,11 @@ class PredibaseChatCompletion(BaseLLM): litellm_params=litellm_params, logger_fn=logger_fn, headers=headers, + timeout=timeout, ) # type: ignore ### SYNC STREAMING - if stream == True: + if stream is True: response = requests.post( completion_url, headers=headers, @@ -420,7 +455,6 @@ class PredibaseChatCompletion(BaseLLM): headers=headers, data=json.dumps(data), ) - return self.process_response( model=model, response=response, @@ -448,16 +482,26 @@ class PredibaseChatCompletion(BaseLLM): stream, data: dict, optional_params: dict, + timeout: Union[float, httpx.Timeout], litellm_params=None, logger_fn=None, headers={}, ) -> ModelResponse: - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) - ) - response = await self.async_handler.post( - api_base, headers=headers, data=json.dumps(data) - ) + + async_handler = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=timeout)) + try: + response = await async_handler.post( + api_base, headers=headers, data=json.dumps(data) + ) + except httpx.HTTPStatusError as e: + raise PredibaseError( + status_code=e.response.status_code, + message="HTTPStatusError - {}".format(e.response.text), + ) + except Exception as e: + raise PredibaseError( + status_code=500, message="{}\n{}".format(str(e), traceback.format_exc()) + ) return self.process_response( model=model, response=response, @@ -483,31 +527,25 @@ class PredibaseChatCompletion(BaseLLM): api_key, logging_obj, data: dict, + timeout: Union[float, httpx.Timeout], optional_params=None, litellm_params=None, logger_fn=None, headers={}, ) -> CustomStreamWrapper: - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) - ) data["stream"] = True - response = await self.async_handler.post( - url=api_base, - headers=headers, - data=json.dumps(data), - stream=True, - ) - - if response.status_code != 200: - raise PredibaseError( - status_code=response.status_code, message=response.text - ) - - completion_stream = response.aiter_lines() streamwrapper = CustomStreamWrapper( - completion_stream=completion_stream, + completion_stream=None, + make_call=partial( + make_call, + api_base=api_base, + headers=headers, + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + ), model=model, custom_llm_provider="predibase", logging_obj=logging_obj, diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 41ecb486c..6bf03b52d 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -3,14 +3,7 @@ import requests, traceback import json, re, xml.etree.ElementTree as ET from jinja2 import Template, exceptions, meta, BaseLoader from jinja2.sandbox import ImmutableSandboxedEnvironment -from typing import ( - Any, - List, - Mapping, - MutableMapping, - Optional, - Sequence, -) +from typing import Any, List, Mapping, MutableMapping, Optional, Sequence, Tuple import litellm import litellm.types from litellm.types.completion import ( @@ -24,7 +17,7 @@ from litellm.types.completion import ( import litellm.types.llms from litellm.types.llms.anthropic import * import uuid - +from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock import litellm.types.llms.vertex_ai @@ -833,7 +826,7 @@ def anthropic_messages_pt_xml(messages: list): ) # either string or none if messages[msg_i].get( "tool_calls", [] - ): # support assistant tool invoke convertion + ): # support assistant tool invoke conversion assistant_text += convert_to_anthropic_tool_invoke_xml( # type: ignore messages[msg_i]["tool_calls"] ) @@ -1224,7 +1217,7 @@ def anthropic_messages_pt(messages: list): if messages[msg_i].get( "tool_calls", [] - ): # support assistant tool invoke convertion + ): # support assistant tool invoke conversion assistant_content.extend( convert_to_anthropic_tool_invoke(messages[msg_i]["tool_calls"]) ) @@ -1460,9 +1453,7 @@ def _load_image_from_url(image_url): try: from PIL import Image except: - raise Exception( - "gemini image conversion failed please run `pip install Pillow`" - ) + raise Exception("image conversion failed please run `pip install Pillow`") from io import BytesIO try: @@ -1613,6 +1604,380 @@ def azure_text_pt(messages: list): return prompt +###### AMAZON BEDROCK ####### + +from litellm.types.llms.bedrock import ( + ToolResultContentBlock as BedrockToolResultContentBlock, + ToolResultBlock as BedrockToolResultBlock, + ToolConfigBlock as BedrockToolConfigBlock, + ToolUseBlock as BedrockToolUseBlock, + ImageSourceBlock as BedrockImageSourceBlock, + ImageBlock as BedrockImageBlock, + ContentBlock as BedrockContentBlock, + ToolInputSchemaBlock as BedrockToolInputSchemaBlock, + ToolSpecBlock as BedrockToolSpecBlock, + ToolBlock as BedrockToolBlock, + ToolChoiceValuesBlock as BedrockToolChoiceValuesBlock, +) + + +def get_image_details(image_url) -> Tuple[str, str]: + try: + import base64 + + # Send a GET request to the image URL + response = requests.get(image_url) + response.raise_for_status() # Raise an exception for HTTP errors + + # Check the response's content type to ensure it is an image + content_type = response.headers.get("content-type") + if not content_type or "image" not in content_type: + raise ValueError( + f"URL does not point to a valid image (content-type: {content_type})" + ) + + # Convert the image content to base64 bytes + base64_bytes = base64.b64encode(response.content).decode("utf-8") + + # Get mime-type + mime_type = content_type.split("/")[ + 1 + ] # Extract mime-type from content-type header + + return base64_bytes, mime_type + + except requests.RequestException as e: + raise Exception(f"Request failed: {e}") + except Exception as e: + raise e + + +def _process_bedrock_converse_image_block(image_url: str) -> BedrockImageBlock: + if "base64" in image_url: + # Case 1: Images with base64 encoding + import base64, re + + # base 64 is passed as data:image/jpeg;base64, + image_metadata, img_without_base_64 = image_url.split(",") + + # read mime_type from img_without_base_64=data:image/jpeg;base64 + # Extract MIME type using regular expression + mime_type_match = re.match(r"data:(.*?);base64", image_metadata) + if mime_type_match: + mime_type = mime_type_match.group(1) + image_format = mime_type.split("/")[1] + else: + mime_type = "image/jpeg" + image_format = "jpeg" + _blob = BedrockImageSourceBlock(bytes=img_without_base_64) + supported_image_formats = ( + litellm.AmazonConverseConfig().get_supported_image_types() + ) + if image_format in supported_image_formats: + return BedrockImageBlock(source=_blob, format=image_format) # type: ignore + else: + # Handle the case when the image format is not supported + raise ValueError( + "Unsupported image format: {}. Supported formats: {}".format( + image_format, supported_image_formats + ) + ) + elif "https:/" in image_url: + # Case 2: Images with direct links + image_bytes, image_format = get_image_details(image_url) + _blob = BedrockImageSourceBlock(bytes=image_bytes) + supported_image_formats = ( + litellm.AmazonConverseConfig().get_supported_image_types() + ) + if image_format in supported_image_formats: + return BedrockImageBlock(source=_blob, format=image_format) # type: ignore + else: + # Handle the case when the image format is not supported + raise ValueError( + "Unsupported image format: {}. Supported formats: {}".format( + image_format, supported_image_formats + ) + ) + else: + raise ValueError( + "Unsupported image type. Expected either image url or base64 encoded string - \ + e.g. 'data:image/jpeg;base64,'" + ) + + +def _convert_to_bedrock_tool_call_invoke( + tool_calls: list, +) -> List[BedrockContentBlock]: + """ + OpenAI tool invokes: + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\n\"location\": \"Boston, MA\"\n}" + } + } + ] + }, + """ + """ + Bedrock tool invokes: + [ + { + "role": "assistant", + "toolUse": { + "input": {"location": "Boston, MA", ..}, + "name": "get_current_weather", + "toolUseId": "call_abc123" + } + } + ] + """ + """ + - json.loads argument + - extract name + - extract id + """ + + try: + _parts_list: List[BedrockContentBlock] = [] + for tool in tool_calls: + if "function" in tool: + id = tool["id"] + name = tool["function"].get("name", "") + arguments = tool["function"].get("arguments", "") + arguments_dict = json.loads(arguments) + bedrock_tool = BedrockToolUseBlock( + input=arguments_dict, name=name, toolUseId=id + ) + bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool) + _parts_list.append(bedrock_content_block) + return _parts_list + except Exception as e: + raise Exception( + "Unable to convert openai tool calls={} to bedrock tool calls. Received error={}".format( + tool_calls, str(e) + ) + ) + + +def _convert_to_bedrock_tool_call_result( + message: dict, +) -> BedrockMessageBlock: + """ + OpenAI message with a tool result looks like: + { + "tool_call_id": "tool_1", + "role": "tool", + "name": "get_current_weather", + "content": "function result goes here", + }, + + OpenAI message with a function call result looks like: + { + "role": "function", + "name": "get_current_weather", + "content": "function result goes here", + } + """ + """ + Bedrock result looks like this: + { + "role": "user", + "content": [ + { + "toolResult": { + "toolUseId": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", + "content": [ + { + "json": { + "song": "Elemental Hotel", + "artist": "8 Storey Hike" + } + } + ] + } + } + ] + } + """ + """ + - + """ + content = message.get("content", "") + name = message.get("name", "") + id = message.get("tool_call_id", str(uuid.uuid4())) + + tool_result_content_block = BedrockToolResultContentBlock(text=content) + tool_result = BedrockToolResultBlock( + content=[tool_result_content_block], + toolUseId=id, + ) + content_block = BedrockContentBlock(toolResult=tool_result) + + return BedrockMessageBlock(role="user", content=[content_block]) + + +def _bedrock_converse_messages_pt(messages: List) -> List[BedrockMessageBlock]: + """ + Converts given messages from OpenAI format to Bedrock format + + - Roles must alternate b/w 'user' and 'model' (same as anthropic -> merge consecutive roles) + - Please ensure that function response turn comes immediately after a function call turn + """ + + contents: List[BedrockMessageBlock] = [] + msg_i = 0 + while msg_i < len(messages): + user_content: List[BedrockContentBlock] = [] + init_msg_i = msg_i + ## MERGE CONSECUTIVE USER CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] == "user": + if isinstance(messages[msg_i]["content"], list): + _parts: List[BedrockContentBlock] = [] + for element in messages[msg_i]["content"]: + if isinstance(element, dict): + if element["type"] == "text": + _part = BedrockContentBlock(text=element["text"]) + _parts.append(_part) + elif element["type"] == "image_url": + image_url = element["image_url"]["url"] + _part = _process_bedrock_converse_image_block( # type: ignore + image_url=image_url + ) + _parts.append(BedrockContentBlock(image=_part)) # type: ignore + user_content.extend(_parts) + else: + _part = BedrockContentBlock(text=messages[msg_i]["content"]) + user_content.append(_part) + + msg_i += 1 + + if user_content: + contents.append(BedrockMessageBlock(role="user", content=user_content)) + assistant_content: List[BedrockContentBlock] = [] + ## MERGE CONSECUTIVE ASSISTANT CONTENT ## + while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": + if isinstance(messages[msg_i]["content"], list): + assistants_parts: List[BedrockContentBlock] = [] + for element in messages[msg_i]["content"]: + if isinstance(element, dict): + if element["type"] == "text": + assistants_part = BedrockContentBlock(text=element["text"]) + assistants_parts.append(assistants_part) + elif element["type"] == "image_url": + image_url = element["image_url"]["url"] + assistants_part = _process_bedrock_converse_image_block( # type: ignore + image_url=image_url + ) + assistants_parts.append( + BedrockContentBlock(image=assistants_part) # type: ignore + ) + assistant_content.extend(assistants_parts) + elif messages[msg_i].get( + "tool_calls", [] + ): # support assistant tool invoke convertion + assistant_content.extend( + _convert_to_bedrock_tool_call_invoke(messages[msg_i]["tool_calls"]) + ) + else: + assistant_text = ( + messages[msg_i].get("content") or "" + ) # either string or none + if assistant_text: + assistant_content.append(BedrockContentBlock(text=assistant_text)) + + msg_i += 1 + + if assistant_content: + contents.append( + BedrockMessageBlock(role="assistant", content=assistant_content) + ) + + ## APPEND TOOL CALL MESSAGES ## + if msg_i < len(messages) and messages[msg_i]["role"] == "tool": + tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i]) + contents.append(tool_call_result) + msg_i += 1 + if msg_i == init_msg_i: # prevent infinite loops + raise Exception( + "Invalid Message passed in - {}. File an issue https://github.com/BerriAI/litellm/issues".format( + messages[msg_i] + ) + ) + + return contents + + +def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]: + """ + OpenAI tools looks like: + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + """ + """ + Bedrock toolConfig looks like: + "tools": [ + { + "toolSpec": { + "name": "top_song", + "description": "Get the most popular song played on a radio station.", + "inputSchema": { + "json": { + "type": "object", + "properties": { + "sign": { + "type": "string", + "description": "The call sign for the radio station for which you want the most popular song. Example calls signs are WZPZ, and WKRP." + } + }, + "required": [ + "sign" + ] + } + } + } + } + ] + """ + tool_block_list: List[BedrockToolBlock] = [] + for tool in tools: + parameters = tool.get("function", {}).get("parameters", None) + name = tool.get("function", {}).get("name", "") + description = tool.get("function", {}).get("description", "") + tool_input_schema = BedrockToolInputSchemaBlock(json=parameters) + tool_spec = BedrockToolSpecBlock( + inputSchema=tool_input_schema, name=name, description=description + ) + tool_block = BedrockToolBlock(toolSpec=tool_spec) + tool_block_list.append(tool_block) + + return tool_block_list + + # Function call template def function_call_prompt(messages: list, functions: list): function_prompt = """Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}} The following functions are available to you:""" diff --git a/litellm/llms/replicate.py b/litellm/llms/replicate.py index 386d24f59..ce62e51e9 100644 --- a/litellm/llms/replicate.py +++ b/litellm/llms/replicate.py @@ -251,7 +251,7 @@ async def async_handle_prediction_response( logs = "" while True and (status not in ["succeeded", "failed", "canceled"]): print_verbose(f"replicate: polling endpoint: {prediction_url}") - await asyncio.sleep(0.5) + await asyncio.sleep(0.5) # prevent replicate rate limit errors response = await http_handler.get(prediction_url, headers=headers) if response.status_code == 200: response_data = response.json() diff --git a/litellm/llms/vertex_ai.py b/litellm/llms/vertex_ai.py index dc185aef9..bd9cfaa8d 100644 --- a/litellm/llms/vertex_ai.py +++ b/litellm/llms/vertex_ai.py @@ -3,7 +3,7 @@ import json from enum import Enum import requests # type: ignore import time -from typing import Callable, Optional, Union, List, Literal +from typing import Callable, Optional, Union, List, Literal, Any from litellm.utils import ModelResponse, Usage, CustomStreamWrapper, map_finish_reason import litellm, uuid import httpx, inspect # type: ignore @@ -12,6 +12,7 @@ from litellm.llms.prompt_templates.factory import ( convert_to_gemini_tool_call_result, convert_to_gemini_tool_call_invoke, ) +from litellm.types.files import get_file_mime_type_for_file_type, get_file_type_from_extension, is_gemini_1_5_accepted_file_type, is_video_file_type class VertexAIError(Exception): @@ -297,24 +298,31 @@ def _convert_gemini_role(role: str) -> Literal["user", "model"]: def _process_gemini_image(image_url: str) -> PartType: try: + # GCS URIs if "gs://" in image_url: - # Case 1: Images with Cloud Storage URIs - # The supported MIME types for images include image/png and image/jpeg. - part_mime = "image/png" if "png" in image_url else "image/jpeg" - _file_data = FileDataType(mime_type=part_mime, file_uri=image_url) - return PartType(file_data=_file_data) + # Figure out file type + extension_with_dot = os.path.splitext(image_url)[-1] # Ex: ".png" + extension = extension_with_dot[1:] # Ex: "png" + + file_type = get_file_type_from_extension(extension) + + # Validate the file type is supported by Gemini + if not is_gemini_1_5_accepted_file_type(file_type): + raise Exception(f"File type not supported by gemini - {file_type}") + + mime_type = get_file_mime_type_for_file_type(file_type) + file_data = FileDataType(mime_type=mime_type, file_uri=image_url) + + return PartType(file_data=file_data) + + # Direct links elif "https:/" in image_url: - # Case 2: Images with direct links image = _load_image_from_url(image_url) _blob = BlobType(data=image.data, mime_type=image._mime_type) return PartType(inline_data=_blob) - elif ".mp4" in image_url and "gs://" in image_url: - # Case 3: Videos with Cloud Storage URIs - part_mime = "video/mp4" - _file_data = FileDataType(mime_type=part_mime, file_uri=image_url) - return PartType(file_data=_file_data) + + # Base64 encoding elif "base64" in image_url: - # Case 4: Images with base64 encoding import base64, re # base 64 is passed as data:image/jpeg;base64, @@ -390,7 +398,7 @@ def _gemini_convert_messages_with_history(messages: list) -> List[ContentType]: assistant_content.extend(_parts) elif messages[msg_i].get( "tool_calls", [] - ): # support assistant tool invoke convertion + ): # support assistant tool invoke conversion assistant_content.extend( convert_to_gemini_tool_call_invoke(messages[msg_i]["tool_calls"]) ) @@ -421,110 +429,17 @@ def _gemini_convert_messages_with_history(messages: list) -> List[ContentType]: return contents -def _gemini_vision_convert_messages(messages: list): - """ - Converts given messages for GPT-4 Vision to Gemini format. +def _get_client_cache_key(model: str, vertex_project: str, vertex_location: str): + _cache_key = f"{model}-{vertex_project}-{vertex_location}" + return _cache_key - Args: - messages (list): The messages to convert. Each message can be a dictionary with a "content" key. The content can be a string or a list of elements. If it is a string, it will be concatenated to the prompt. If it is a list, each element will be processed based on its type: - - If the element is a dictionary with a "type" key equal to "text", its "text" value will be concatenated to the prompt. - - If the element is a dictionary with a "type" key equal to "image_url", its "image_url" value will be added to the list of images. - Returns: - tuple: A tuple containing the prompt (a string) and the processed images (a list of objects representing the images). +def _get_client_from_cache(client_cache_key: str): + return litellm.in_memory_llm_clients_cache.get(client_cache_key, None) - Raises: - VertexAIError: If the import of the 'vertexai' module fails, indicating that 'google-cloud-aiplatform' needs to be installed. - Exception: If any other exception occurs during the execution of the function. - Note: - This function is based on the code from the 'gemini/getting-started/intro_gemini_python.ipynb' notebook in the 'generative-ai' repository on GitHub. - The supported MIME types for images include 'image/png' and 'image/jpeg'. - - Examples: - >>> messages = [ - ... {"content": "Hello, world!"}, - ... {"content": [{"type": "text", "text": "This is a text message."}, {"type": "image_url", "image_url": "example.com/image.png"}]}, - ... ] - >>> _gemini_vision_convert_messages(messages) - ('Hello, world!This is a text message.', [, ]) - """ - try: - import vertexai - except: - raise VertexAIError( - status_code=400, - message="vertexai import failed please run `pip install google-cloud-aiplatform`", - ) - try: - from vertexai.preview.language_models import ( - ChatModel, - CodeChatModel, - InputOutputTextPair, - ) - from vertexai.language_models import TextGenerationModel, CodeGenerationModel - from vertexai.preview.generative_models import ( - GenerativeModel, - Part, - GenerationConfig, - Image, - ) - - # given messages for gpt-4 vision, convert them for gemini - # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/getting-started/intro_gemini_python.ipynb - prompt = "" - images = [] - for message in messages: - if isinstance(message["content"], str): - prompt += message["content"] - elif isinstance(message["content"], list): - # see https://docs.litellm.ai/docs/providers/openai#openai-vision-models - for element in message["content"]: - if isinstance(element, dict): - if element["type"] == "text": - prompt += element["text"] - elif element["type"] == "image_url": - image_url = element["image_url"]["url"] - images.append(image_url) - # processing images passed to gemini - processed_images = [] - for img in images: - if "gs://" in img: - # Case 1: Images with Cloud Storage URIs - # The supported MIME types for images include image/png and image/jpeg. - part_mime = "image/png" if "png" in img else "image/jpeg" - google_clooud_part = Part.from_uri(img, mime_type=part_mime) - processed_images.append(google_clooud_part) - elif "https:/" in img: - # Case 2: Images with direct links - image = _load_image_from_url(img) - processed_images.append(image) - elif ".mp4" in img and "gs://" in img: - # Case 3: Videos with Cloud Storage URIs - part_mime = "video/mp4" - google_clooud_part = Part.from_uri(img, mime_type=part_mime) - processed_images.append(google_clooud_part) - elif "base64" in img: - # Case 4: Images with base64 encoding - import base64, re - - # base 64 is passed as data:image/jpeg;base64, - image_metadata, img_without_base_64 = img.split(",") - - # read mime_type from img_without_base_64=data:image/jpeg;base64 - # Extract MIME type using regular expression - mime_type_match = re.match(r"data:(.*?);base64", image_metadata) - - if mime_type_match: - mime_type = mime_type_match.group(1) - else: - mime_type = "image/jpeg" - decoded_img = base64.b64decode(img_without_base_64) - processed_image = Part.from_data(data=decoded_img, mime_type=mime_type) - processed_images.append(processed_image) - return prompt, processed_images - except Exception as e: - raise e +def _set_client_in_cache(client_cache_key: str, vertex_llm_model: Any): + litellm.in_memory_llm_clients_cache[client_cache_key] = vertex_llm_model def completion( @@ -580,23 +495,32 @@ def completion( print_verbose( f"VERTEX AI: vertex_project={vertex_project}; vertex_location={vertex_location}" ) - if vertex_credentials is not None and isinstance(vertex_credentials, str): - import google.oauth2.service_account - json_obj = json.loads(vertex_credentials) + _cache_key = _get_client_cache_key( + model=model, vertex_project=vertex_project, vertex_location=vertex_location + ) + _vertex_llm_model_object = _get_client_from_cache(client_cache_key=_cache_key) - creds = google.oauth2.service_account.Credentials.from_service_account_info( - json_obj, - scopes=["https://www.googleapis.com/auth/cloud-platform"], + if _vertex_llm_model_object is None: + if vertex_credentials is not None and isinstance(vertex_credentials, str): + import google.oauth2.service_account + + json_obj = json.loads(vertex_credentials) + + creds = ( + google.oauth2.service_account.Credentials.from_service_account_info( + json_obj, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + ) + else: + creds, _ = google.auth.default(quota_project_id=vertex_project) + print_verbose( + f"VERTEX AI: creds={creds}; google application credentials: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}" + ) + vertexai.init( + project=vertex_project, location=vertex_location, credentials=creds ) - else: - creds, _ = google.auth.default(quota_project_id=vertex_project) - print_verbose( - f"VERTEX AI: creds={creds}; google application credentials: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}" - ) - vertexai.init( - project=vertex_project, location=vertex_location, credentials=creds - ) ## Load Config config = litellm.VertexAIConfig.get_config() @@ -620,9 +544,9 @@ def completion( prompt = " ".join( [ - message["content"] + message.get("content") for message in messages - if isinstance(message["content"], str) + if isinstance(message.get("content", None), str) ] ) @@ -639,23 +563,27 @@ def completion( model in litellm.vertex_language_models or model in litellm.vertex_vision_models ): - llm_model = GenerativeModel(model) + llm_model = _vertex_llm_model_object or GenerativeModel(model) mode = "vision" request_str += f"llm_model = GenerativeModel({model})\n" elif model in litellm.vertex_chat_models: - llm_model = ChatModel.from_pretrained(model) + llm_model = _vertex_llm_model_object or ChatModel.from_pretrained(model) mode = "chat" request_str += f"llm_model = ChatModel.from_pretrained({model})\n" elif model in litellm.vertex_text_models: - llm_model = TextGenerationModel.from_pretrained(model) + llm_model = _vertex_llm_model_object or TextGenerationModel.from_pretrained( + model + ) mode = "text" request_str += f"llm_model = TextGenerationModel.from_pretrained({model})\n" elif model in litellm.vertex_code_text_models: - llm_model = CodeGenerationModel.from_pretrained(model) + llm_model = _vertex_llm_model_object or CodeGenerationModel.from_pretrained( + model + ) mode = "text" request_str += f"llm_model = CodeGenerationModel.from_pretrained({model})\n" elif model in litellm.vertex_code_chat_models: # vertex_code_llm_models - llm_model = CodeChatModel.from_pretrained(model) + llm_model = _vertex_llm_model_object or CodeChatModel.from_pretrained(model) mode = "chat" request_str += f"llm_model = CodeChatModel.from_pretrained({model})\n" elif model == "private": @@ -1034,6 +962,15 @@ async def async_completion( tools=tools, ) + _cache_key = _get_client_cache_key( + model=model, + vertex_project=vertex_project, + vertex_location=vertex_location, + ) + _set_client_in_cache( + client_cache_key=_cache_key, vertex_llm_model=llm_model + ) + if tools is not None and bool( getattr(response.candidates[0].content.parts[0], "function_call", None) ): diff --git a/litellm/main.py b/litellm/main.py index 37fc1db8f..2c906e990 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -79,7 +79,7 @@ from .llms.anthropic import AnthropicChatCompletion from .llms.anthropic_text import AnthropicTextCompletion from .llms.huggingface_restapi import Huggingface from .llms.predibase import PredibaseChatCompletion -from .llms.bedrock_httpx import BedrockLLM +from .llms.bedrock_httpx import BedrockLLM, BedrockConverseLLM from .llms.vertex_httpx import VertexLLM from .llms.triton import TritonChatCompletion from .llms.prompt_templates.factory import ( @@ -92,6 +92,7 @@ import tiktoken from concurrent.futures import ThreadPoolExecutor from typing import Callable, List, Optional, Dict, Union, Mapping from .caching import enable_cache, disable_cache, update_cache +from .types.llms.openai import HttpxBinaryResponseContent encoding = tiktoken.get_encoding("cl100k_base") from litellm.utils import ( @@ -121,6 +122,7 @@ huggingface = Huggingface() predibase_chat_completions = PredibaseChatCompletion() triton_chat_completions = TritonChatCompletion() bedrock_chat_completion = BedrockLLM() +bedrock_converse_chat_completion = BedrockConverseLLM() vertex_chat_completion = VertexLLM() ####### COMPLETION ENDPOINTS ################ @@ -223,7 +225,7 @@ async def acompletion( extra_headers: Optional[dict] = None, # Optional liteLLM function params **kwargs, -): +) -> Union[ModelResponse, CustomStreamWrapper]: """ Asynchronously executes a litellm.completion() call for any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly) @@ -294,6 +296,7 @@ async def acompletion( "api_version": api_version, "api_key": api_key, "model_list": model_list, + "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter } if custom_llm_provider is None: @@ -338,6 +341,8 @@ async def acompletion( if isinstance(init_response, dict) or isinstance( init_response, ModelResponse ): ## CACHING SCENARIO + if isinstance(init_response, dict): + response = ModelResponse(**init_response) response = init_response elif asyncio.iscoroutine(init_response): response = await init_response @@ -360,6 +365,10 @@ async def acompletion( ) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls) return response except Exception as e: + verbose_logger.error( + "litellm.acompletion(): Exception occured - {}".format(str(e)) + ) + verbose_logger.debug(traceback.format_exc()) custom_llm_provider = custom_llm_provider or "openai" raise exception_type( model=model, @@ -423,12 +432,16 @@ def mock_completion( if isinstance(mock_response, openai.APIError): raise mock_response raise litellm.APIError( - status_code=500, # type: ignore - message=str(mock_response), - llm_provider="openai", # type: ignore + status_code=getattr(mock_response, "status_code", 500), # type: ignore + message=getattr(mock_response, "text", str(mock_response)), + llm_provider=getattr(mock_response, "llm_provider", "openai"), # type: ignore model=model, # type: ignore request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), ) + time_delay = kwargs.get("mock_delay", None) + if time_delay is not None: + time.sleep(time_delay) + model_response = ModelResponse(stream=stream) if stream is True: # don't try to access stream object, @@ -459,16 +472,25 @@ def mock_completion( try: _, custom_llm_provider, _, _ = litellm.utils.get_llm_provider(model=model) model_response._hidden_params["custom_llm_provider"] = custom_llm_provider - except: + except Exception: # dont let setting a hidden param block a mock_respose pass + if logging is not None: + logging.post_call( + input=messages, + api_key="my-secret-key", + original_response="my-original-response", + ) return model_response except Exception as e: if isinstance(e, openai.APIError): raise e - traceback.print_exc() + verbose_logger.error( + "litellm.mock_completion(): Exception occured - {}".format(str(e)) + ) + verbose_logger.debug(traceback.format_exc()) raise Exception("Mock completion response failed") @@ -679,6 +701,7 @@ def completion( "region_name", "allowed_model_region", "model_config", + "fastest_response", ] default_params = openai_params + litellm_params @@ -828,6 +851,7 @@ def completion( logprobs=logprobs, top_logprobs=top_logprobs, extra_headers=extra_headers, + api_version=api_version, **non_default_params, ) @@ -878,6 +902,7 @@ def completion( mock_response=mock_response, logging=logging, acompletion=acompletion, + mock_delay=kwargs.get("mock_delay", None), ) if custom_llm_provider == "azure": # azure configs @@ -1924,7 +1949,8 @@ def completion( ) api_base = ( - optional_params.pop("api_base", None) + api_base + or optional_params.pop("api_base", None) or optional_params.pop("base_url", None) or litellm.api_base or get_secret("PREDIBASE_API_BASE") @@ -1952,12 +1978,13 @@ def completion( custom_prompt_dict=custom_prompt_dict, api_key=api_key, tenant_id=tenant_id, + timeout=timeout, ) if ( "stream" in optional_params - and optional_params["stream"] == True - and acompletion == False + and optional_params["stream"] is True + and acompletion is False ): return _model_response response = _model_response @@ -2085,22 +2112,40 @@ def completion( logging_obj=logging, ) else: - response = bedrock_chat_completion.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=encoding, - logging_obj=logging, - extra_headers=extra_headers, - timeout=timeout, - acompletion=acompletion, - client=client, - ) + if model.startswith("anthropic"): + response = bedrock_converse_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + acompletion=acompletion, + client=client, + ) + else: + response = bedrock_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + acompletion=acompletion, + client=client, + ) if optional_params.get("stream", False): ## LOGGING logging.post_call( @@ -2403,6 +2448,7 @@ def completion( "top_k": kwargs.get("top_k", 40), }, }, + verify=litellm.ssl_verify, ) response_json = resp.json() """ @@ -3712,7 +3758,7 @@ async def amoderation(input: str, model: str, api_key: Optional[str] = None, **k ##### Image Generation ####################### @client -async def aimage_generation(*args, **kwargs): +async def aimage_generation(*args, **kwargs) -> ImageResponse: """ Asynchronously calls the `image_generation` function with the given arguments and keyword arguments. @@ -3745,6 +3791,8 @@ async def aimage_generation(*args, **kwargs): if isinstance(init_response, dict) or isinstance( init_response, ImageResponse ): ## CACHING SCENARIO + if isinstance(init_response, dict): + init_response = ImageResponse(**init_response) response = init_response elif asyncio.iscoroutine(init_response): response = await init_response @@ -3780,7 +3828,7 @@ def image_generation( litellm_logging_obj=None, custom_llm_provider=None, **kwargs, -): +) -> ImageResponse: """ Maps the https://api.openai.com/v1/images/generations endpoint. @@ -4112,7 +4160,7 @@ def transcription( or litellm.api_key or litellm.azure_key or get_secret("AZURE_API_KEY") - ) + ) # type: ignore response = azure_chat_completions.audio_transcriptions( model=model, @@ -4129,6 +4177,24 @@ def transcription( max_retries=max_retries, ) elif custom_llm_provider == "openai": + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) # type: ignore + openai.organization = ( + litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) # type: ignore response = openai_chat_completions.audio_transcriptions( model=model, audio_file=file, @@ -4138,6 +4204,139 @@ def transcription( timeout=timeout, logging_obj=litellm_logging_obj, max_retries=max_retries, + api_base=api_base, + api_key=api_key, + ) + return response + + +@client +async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent: + """ + Calls openai tts endpoints. + """ + loop = asyncio.get_event_loop() + model = args[0] if len(args) > 0 else kwargs["model"] + ### PASS ARGS TO Image Generation ### + kwargs["aspeech"] = True + custom_llm_provider = kwargs.get("custom_llm_provider", None) + try: + # Use a partial function to pass your keyword arguments + func = partial(speech, *args, **kwargs) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + + _, custom_llm_provider, _, _ = get_llm_provider( + model=model, api_base=kwargs.get("api_base", None) + ) + + # Await normally + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + # Call the synchronous function using run_in_executor + response = await loop.run_in_executor(None, func_with_context) + return response # type: ignore + except Exception as e: + custom_llm_provider = custom_llm_provider or "openai" + raise exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=args, + extra_kwargs=kwargs, + ) + + +@client +def speech( + model: str, + input: str, + voice: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + organization: Optional[str] = None, + project: Optional[str] = None, + max_retries: Optional[int] = None, + metadata: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + response_format: Optional[str] = None, + speed: Optional[int] = None, + client=None, + headers: Optional[dict] = None, + custom_llm_provider: Optional[str] = None, + aspeech: Optional[bool] = None, + **kwargs, +) -> HttpxBinaryResponseContent: + + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) # type: ignore + + optional_params = {} + if response_format is not None: + optional_params["response_format"] = response_format + if speed is not None: + optional_params["speed"] = speed # type: ignore + + if timeout is None: + timeout = litellm.request_timeout + + if max_retries is None: + max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES + response: Optional[HttpxBinaryResponseContent] = None + if custom_llm_provider == "openai": + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) # type: ignore + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) # type: ignore + + organization = ( + organization + or litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) # type: ignore + + project = ( + project + or litellm.project + or get_secret("OPENAI_PROJECT") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) # type: ignore + + headers = headers or litellm.headers + + response = openai_chat_completions.audio_speech( + model=model, + input=input, + voice=voice, + optional_params=optional_params, + api_key=api_key, + api_base=api_base, + organization=organization, + project=project, + max_retries=max_retries, + timeout=timeout, + client=client, # pass AsyncOpenAI, OpenAI client + aspeech=aspeech, + ) + + if response is None: + raise Exception( + "Unable to map the custom llm provider={} to a known provider={}.".format( + custom_llm_provider, litellm.provider_list + ) ) return response @@ -4170,6 +4369,10 @@ async def ahealth_check( mode = litellm.model_cost[model]["mode"] model, custom_llm_provider, _, _ = get_llm_provider(model=model) + + if model in litellm.model_cost and mode is None: + mode = litellm.model_cost[model]["mode"] + mode = mode or "chat" # default to chat completion calls if custom_llm_provider == "azure": @@ -4260,7 +4463,10 @@ async def ahealth_check( response = {} # args like remaining ratelimit etc. return response except Exception as e: - traceback.print_exc() + verbose_logger.error( + "litellm.ahealth_check(): Exception occured - {}".format(str(e)) + ) + verbose_logger.debug(traceback.format_exc()) stack_trace = traceback.format_exc() if isinstance(stack_trace, str): stack_trace = stack_trace[:1000] @@ -4366,7 +4572,7 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List] def stream_chunk_builder( chunks: list, messages: Optional[list] = None, start_time=None, end_time=None -): +) -> Union[ModelResponse, TextCompletionResponse]: model_response = litellm.ModelResponse() ### SORT CHUNKS BASED ON CREATED ORDER ## print_verbose("Goes into checking if chunk has hiddden created at param") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index aab9c9af1..f2b292c92 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -380,6 +380,18 @@ "output_cost_per_second": 0.0001, "litellm_provider": "azure" }, + "azure/gpt-4o": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -692,8 +704,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.00000015, - "output_cost_per_token": 0.00000046, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000025, "litellm_provider": "mistral", "mode": "chat" }, @@ -701,8 +713,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat" @@ -711,8 +723,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000006, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat" @@ -748,8 +760,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 0.000004, + "output_cost_per_token": 0.000012, "litellm_provider": "mistral", "mode": "chat", "supports_function_calling": true @@ -758,26 +770,63 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000008, - "output_cost_per_token": 0.000024, + "input_cost_per_token": 0.000004, + "output_cost_per_token": 0.000012, "litellm_provider": "mistral", "mode": "chat", "supports_function_calling": true }, + "mistral/open-mistral-7b": { + "max_tokens": 8191, + "max_input_tokens": 32000, + "max_output_tokens": 8191, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000025, + "litellm_provider": "mistral", + "mode": "chat" + }, "mistral/open-mixtral-8x7b": { "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, + "input_cost_per_token": 0.0000007, + "output_cost_per_token": 0.0000007, + "litellm_provider": "mistral", + "mode": "chat", + "supports_function_calling": true + }, + "mistral/open-mixtral-8x22b": { + "max_tokens": 8191, + "max_input_tokens": 64000, + "max_output_tokens": 8191, "input_cost_per_token": 0.000002, "output_cost_per_token": 0.000006, "litellm_provider": "mistral", "mode": "chat", "supports_function_calling": true }, + "mistral/codestral-latest": { + "max_tokens": 8191, + "max_input_tokens": 32000, + "max_output_tokens": 8191, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "litellm_provider": "mistral", + "mode": "chat" + }, + "mistral/codestral-2405": { + "max_tokens": 8191, + "max_input_tokens": 32000, + "max_output_tokens": 8191, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "litellm_provider": "mistral", + "mode": "chat" + }, "mistral/mistral-embed": { "max_tokens": 8192, "max_input_tokens": 8192, - "input_cost_per_token": 0.000000111, + "input_cost_per_token": 0.0000001, "litellm_provider": "mistral", "mode": "embedding" }, @@ -1128,6 +1177,24 @@ "supports_tool_choice": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, + "gemini-1.5-flash-001": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_images_per_prompt": 3000, + "max_videos_per_prompt": 10, + "max_video_length": 1, + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_pdf_size_mb": 30, + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" + }, "gemini-1.5-flash-preview-0514": { "max_tokens": 8192, "max_input_tokens": 1000000, @@ -1146,6 +1213,18 @@ "supports_vision": true, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" }, + "gemini-1.5-pro-001": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.000000625, + "output_cost_per_token": 0.000001875, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" + }, "gemini-1.5-pro-preview-0514": { "max_tokens": 8192, "max_input_tokens": 1000000, @@ -1265,8 +1344,8 @@ "max_tokens": 4096, "max_input_tokens": 200000, "max_output_tokens": 4096, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.0000075, + "input_cost_per_token": 0.000015, + "output_cost_per_token": 0.000075, "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", "supports_function_calling": true, @@ -1421,7 +1500,7 @@ "max_pdf_size_mb": 30, "input_cost_per_token": 0, "output_cost_per_token": 0, - "litellm_provider": "vertex_ai-language-models", + "litellm_provider": "gemini", "mode": "chat", "supports_function_calling": true, "supports_vision": true, @@ -2930,32 +3009,37 @@ "litellm_provider": "sagemaker", "mode": "chat" }, - "together-ai-up-to-3b": { + "together-ai-up-to-4b": { "input_cost_per_token": 0.0000001, "output_cost_per_token": 0.0000001, "litellm_provider": "together_ai" }, - "together-ai-3.1b-7b": { + "together-ai-4.1b-8b": { "input_cost_per_token": 0.0000002, "output_cost_per_token": 0.0000002, "litellm_provider": "together_ai" }, - "together-ai-7.1b-20b": { + "together-ai-8.1b-21b": { "max_tokens": 1000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.0000003, "litellm_provider": "together_ai" }, - "together-ai-20.1b-40b": { + "together-ai-21.1b-41b": { "input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000008, "litellm_provider": "together_ai" }, - "together-ai-40.1b-70b": { + "together-ai-41.1b-80b": { "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "together_ai" }, + "together-ai-81.1b-110b": { + "input_cost_per_token": 0.0000018, + "output_cost_per_token": 0.0000018, + "litellm_provider": "together_ai" + }, "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { "input_cost_per_token": 0.0000006, "output_cost_per_token": 0.0000006, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index 14787d256..fc813d761 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/D_ZUmMtLMPSa4aQQUJtKt/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/48nWsJi-LJrUlOLzcK-Yz/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/D_ZUmMtLMPSa4aQQUJtKt/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/48nWsJi-LJrUlOLzcK-Yz/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/D_ZUmMtLMPSa4aQQUJtKt/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/48nWsJi-LJrUlOLzcK-Yz/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/D_ZUmMtLMPSa4aQQUJtKt/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/48nWsJi-LJrUlOLzcK-Yz/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/131-6a03368053f9d26d.js b/litellm/proxy/_experimental/out/_next/static/chunks/131-6a03368053f9d26d.js new file mode 100644 index 000000000..f6ea1fb19 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/131-6a03368053f9d26d.js @@ -0,0 +1,8 @@ +"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[131],{84174:function(e,t,n){n.d(t,{Z:function(){return s}});var a=n(14749),r=n(64090),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M832 64H296c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h496v688c0 4.4 3.6 8 8 8h56c4.4 0 8-3.6 8-8V96c0-17.7-14.3-32-32-32zM704 192H192c-17.7 0-32 14.3-32 32v530.7c0 8.5 3.4 16.6 9.4 22.6l173.3 173.3c2.2 2.2 4.7 4 7.4 5.5v1.9h4.2c3.5 1.3 7.2 2 11 2H704c17.7 0 32-14.3 32-32V224c0-17.7-14.3-32-32-32zM350 856.2L263.9 770H350v86.2zM664 888H414V746c0-22.1-17.9-40-40-40H232V264h432v624z"}}]},name:"copy",theme:"outlined"},o=n(60688),s=r.forwardRef(function(e,t){return r.createElement(o.Z,(0,a.Z)({},e,{ref:t,icon:i}))})},50459:function(e,t,n){n.d(t,{Z:function(){return s}});var a=n(14749),r=n(64090),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M765.7 486.8L314.9 134.7A7.97 7.97 0 00302 141v77.3c0 4.9 2.3 9.6 6.1 12.6l360 281.1-360 281.1c-3.9 3-6.1 7.7-6.1 12.6V883c0 6.7 7.7 10.4 12.9 6.3l450.8-352.1a31.96 31.96 0 000-50.4z"}}]},name:"right",theme:"outlined"},o=n(60688),s=r.forwardRef(function(e,t){return r.createElement(o.Z,(0,a.Z)({},e,{ref:t,icon:i}))})},92836:function(e,t,n){n.d(t,{Z:function(){return p}});var a=n(69703),r=n(80991),i=n(2898),o=n(99250),s=n(65492),l=n(64090),c=n(41608),d=n(50027);n(18174),n(21871),n(41213);let u=(0,s.fn)("Tab"),p=l.forwardRef((e,t)=>{let{icon:n,className:p,children:g}=e,m=(0,a._T)(e,["icon","className","children"]),b=(0,l.useContext)(c.O),f=(0,l.useContext)(d.Z);return l.createElement(r.O,Object.assign({ref:t,className:(0,o.q)(u("root"),"flex whitespace-nowrap truncate max-w-xs outline-none focus:ring-0 text-tremor-default transition duration-100",f?(0,s.bM)(f,i.K.text).selectTextColor:"solid"===b?"ui-selected:text-tremor-content-emphasis dark:ui-selected:text-dark-tremor-content-emphasis":"ui-selected:text-tremor-brand dark:ui-selected:text-dark-tremor-brand",function(e,t){switch(e){case"line":return(0,o.q)("ui-selected:border-b-2 hover:border-b-2 border-transparent transition duration-100 -mb-px px-2 py-2","hover:border-tremor-content hover:text-tremor-content-emphasis text-tremor-content","dark:hover:border-dark-tremor-content-emphasis dark:hover:text-dark-tremor-content-emphasis dark:text-dark-tremor-content",t?(0,s.bM)(t,i.K.border).selectBorderColor:"ui-selected:border-tremor-brand dark:ui-selected:border-dark-tremor-brand");case"solid":return(0,o.q)("border-transparent border rounded-tremor-small px-2.5 py-1","ui-selected:border-tremor-border ui-selected:bg-tremor-background ui-selected:shadow-tremor-input hover:text-tremor-content-emphasis ui-selected:text-tremor-brand","dark:ui-selected:border-dark-tremor-border dark:ui-selected:bg-dark-tremor-background dark:ui-selected:shadow-dark-tremor-input dark:hover:text-dark-tremor-content-emphasis dark:ui-selected:text-dark-tremor-brand",t?(0,s.bM)(t,i.K.text).selectTextColor:"text-tremor-content dark:text-dark-tremor-content")}}(b,f),p)},m),n?l.createElement(n,{className:(0,o.q)(u("icon"),"flex-none h-5 w-5",g?"mr-2":"")}):null,g?l.createElement("span",null,g):null)});p.displayName="Tab"},26734:function(e,t,n){n.d(t,{Z:function(){return c}});var a=n(69703),r=n(80991),i=n(99250),o=n(65492),s=n(64090);let l=(0,o.fn)("TabGroup"),c=s.forwardRef((e,t)=>{let{defaultIndex:n,index:o,onIndexChange:c,children:d,className:u}=e,p=(0,a._T)(e,["defaultIndex","index","onIndexChange","children","className"]);return s.createElement(r.O.Group,Object.assign({as:"div",ref:t,defaultIndex:n,selectedIndex:o,onChange:c,className:(0,i.q)(l("root"),"w-full",u)},p),d)});c.displayName="TabGroup"},41608:function(e,t,n){n.d(t,{O:function(){return c},Z:function(){return u}});var a=n(69703),r=n(64090),i=n(50027);n(18174),n(21871),n(41213);var o=n(80991),s=n(99250);let l=(0,n(65492).fn)("TabList"),c=(0,r.createContext)("line"),d={line:(0,s.q)("flex border-b space-x-4","border-tremor-border","dark:border-dark-tremor-border"),solid:(0,s.q)("inline-flex p-0.5 rounded-tremor-default space-x-1.5","bg-tremor-background-subtle","dark:bg-dark-tremor-background-subtle")},u=r.forwardRef((e,t)=>{let{color:n,variant:u="line",children:p,className:g}=e,m=(0,a._T)(e,["color","variant","children","className"]);return r.createElement(o.O.List,Object.assign({ref:t,className:(0,s.q)(l("root"),"justify-start overflow-x-clip",d[u],g)},m),r.createElement(c.Provider,{value:u},r.createElement(i.Z.Provider,{value:n},p)))});u.displayName="TabList"},32126:function(e,t,n){n.d(t,{Z:function(){return d}});var a=n(69703);n(50027);var r=n(18174);n(21871);var i=n(41213),o=n(99250),s=n(65492),l=n(64090);let c=(0,s.fn)("TabPanel"),d=l.forwardRef((e,t)=>{let{children:n,className:s}=e,d=(0,a._T)(e,["children","className"]),{selectedValue:u}=(0,l.useContext)(i.Z),p=u===(0,l.useContext)(r.Z);return l.createElement("div",Object.assign({ref:t,className:(0,o.q)(c("root"),"w-full mt-2",p?"":"hidden",s),"aria-selected":p?"true":"false"},d),n)});d.displayName="TabPanel"},23682:function(e,t,n){n.d(t,{Z:function(){return u}});var a=n(69703),r=n(80991);n(50027);var i=n(18174);n(21871);var o=n(41213),s=n(99250),l=n(65492),c=n(64090);let d=(0,l.fn)("TabPanels"),u=c.forwardRef((e,t)=>{let{children:n,className:l}=e,u=(0,a._T)(e,["children","className"]);return c.createElement(r.O.Panels,Object.assign({as:"div",ref:t,className:(0,s.q)(d("root"),"w-full",l)},u),e=>{let{selectedIndex:t}=e;return c.createElement(o.Z.Provider,{value:{selectedValue:t}},c.Children.map(n,(e,t)=>c.createElement(i.Z.Provider,{value:t},e)))})});u.displayName="TabPanels"},50027:function(e,t,n){n.d(t,{Z:function(){return i}});var a=n(64090),r=n(54942);n(99250);let i=(0,a.createContext)(r.fr.Blue)},18174:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(64090).createContext)(0)},21871:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(64090).createContext)(void 0)},41213:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(64090).createContext)({selectedValue:void 0,handleValueChange:void 0})},21467:function(e,t,n){n.d(t,{i:function(){return s}});var a=n(64090),r=n(44329),i=n(54165),o=n(57499);function s(e){return t=>a.createElement(i.ZP,{theme:{token:{motion:!1,zIndexPopupBase:0}}},a.createElement(e,Object.assign({},t)))}t.Z=(e,t,n,i)=>s(s=>{let{prefixCls:l,style:c}=s,d=a.useRef(null),[u,p]=a.useState(0),[g,m]=a.useState(0),[b,f]=(0,r.Z)(!1,{value:s.open}),{getPrefixCls:E}=a.useContext(o.E_),h=E(t||"select",l);a.useEffect(()=>{if(f(!0),"undefined"!=typeof ResizeObserver){let e=new ResizeObserver(e=>{let t=e[0].target;p(t.offsetHeight+8),m(t.offsetWidth)}),t=setInterval(()=>{var a;let r=n?".".concat(n(h)):".".concat(h,"-dropdown"),i=null===(a=d.current)||void 0===a?void 0:a.querySelector(r);i&&(clearInterval(t),e.observe(i))},10);return()=>{clearInterval(t),e.disconnect()}}},[]);let S=Object.assign(Object.assign({},s),{style:Object.assign(Object.assign({},c),{margin:0}),open:b,visible:b,getPopupContainer:()=>d.current});return i&&(S=i(S)),a.createElement("div",{ref:d,style:{paddingBottom:u,position:"relative",minWidth:g}},a.createElement(e,Object.assign({},S)))})},99129:function(e,t,n){let a;n.d(t,{Z:function(){return eY}});var r=n(63787),i=n(64090),o=n(37274),s=n(57499),l=n(54165),c=n(99537),d=n(77136),u=n(20653),p=n(40388),g=n(16480),m=n.n(g),b=n(51761),f=n(47387),E=n(70595),h=n(24750),S=n(89211),y=n(1861),T=n(51350),A=e=>{let{type:t,children:n,prefixCls:a,buttonProps:r,close:o,autoFocus:s,emitEvent:l,isSilent:c,quitOnNullishReturnValue:d,actionFn:u}=e,p=i.useRef(!1),g=i.useRef(null),[m,b]=(0,S.Z)(!1),f=function(){null==o||o.apply(void 0,arguments)};i.useEffect(()=>{let e=null;return s&&(e=setTimeout(()=>{var e;null===(e=g.current)||void 0===e||e.focus()})),()=>{e&&clearTimeout(e)}},[]);let E=e=>{e&&e.then&&(b(!0),e.then(function(){b(!1,!0),f.apply(void 0,arguments),p.current=!1},e=>{if(b(!1,!0),p.current=!1,null==c||!c())return Promise.reject(e)}))};return i.createElement(y.ZP,Object.assign({},(0,T.nx)(t),{onClick:e=>{let t;if(!p.current){if(p.current=!0,!u){f();return}if(l){var n;if(t=u(e),d&&!((n=t)&&n.then)){p.current=!1,f(e);return}}else if(u.length)t=u(o),p.current=!1;else if(!(t=u())){f();return}E(t)}},loading:m,prefixCls:a},r,{ref:g}),n)};let R=i.createContext({}),{Provider:I}=R;var N=()=>{let{autoFocusButton:e,cancelButtonProps:t,cancelTextLocale:n,isSilent:a,mergedOkCancel:r,rootPrefixCls:o,close:s,onCancel:l,onConfirm:c}=(0,i.useContext)(R);return r?i.createElement(A,{isSilent:a,actionFn:l,close:function(){null==s||s.apply(void 0,arguments),null==c||c(!1)},autoFocus:"cancel"===e,buttonProps:t,prefixCls:"".concat(o,"-btn")},n):null},_=()=>{let{autoFocusButton:e,close:t,isSilent:n,okButtonProps:a,rootPrefixCls:r,okTextLocale:o,okType:s,onConfirm:l,onOk:c}=(0,i.useContext)(R);return i.createElement(A,{isSilent:n,type:s||"primary",actionFn:c,close:function(){null==t||t.apply(void 0,arguments),null==l||l(!0)},autoFocus:"ok"===e,buttonProps:a,prefixCls:"".concat(r,"-btn")},o)},v=n(81303),w=n(14749),k=n(80406),C=n(88804),O=i.createContext({}),x=n(5239),L=n(31506),D=n(91010),P=n(4295),M=n(72480);function F(e,t,n){var a=t;return!a&&n&&(a="".concat(e,"-").concat(n)),a}function U(e,t){var n=e["page".concat(t?"Y":"X","Offset")],a="scroll".concat(t?"Top":"Left");if("number"!=typeof n){var r=e.document;"number"!=typeof(n=r.documentElement[a])&&(n=r.body[a])}return n}var B=n(49367),G=n(74084),$=i.memo(function(e){return e.children},function(e,t){return!t.shouldUpdate}),H={width:0,height:0,overflow:"hidden",outline:"none"},z=i.forwardRef(function(e,t){var n,a,r,o=e.prefixCls,s=e.className,l=e.style,c=e.title,d=e.ariaId,u=e.footer,p=e.closable,g=e.closeIcon,b=e.onClose,f=e.children,E=e.bodyStyle,h=e.bodyProps,S=e.modalRender,y=e.onMouseDown,T=e.onMouseUp,A=e.holderRef,R=e.visible,I=e.forceRender,N=e.width,_=e.height,v=e.classNames,k=e.styles,C=i.useContext(O).panel,L=(0,G.x1)(A,C),D=(0,i.useRef)(),P=(0,i.useRef)();i.useImperativeHandle(t,function(){return{focus:function(){var e;null===(e=D.current)||void 0===e||e.focus()},changeActive:function(e){var t=document.activeElement;e&&t===P.current?D.current.focus():e||t!==D.current||P.current.focus()}}});var M={};void 0!==N&&(M.width=N),void 0!==_&&(M.height=_),u&&(n=i.createElement("div",{className:m()("".concat(o,"-footer"),null==v?void 0:v.footer),style:(0,x.Z)({},null==k?void 0:k.footer)},u)),c&&(a=i.createElement("div",{className:m()("".concat(o,"-header"),null==v?void 0:v.header),style:(0,x.Z)({},null==k?void 0:k.header)},i.createElement("div",{className:"".concat(o,"-title"),id:d},c))),p&&(r=i.createElement("button",{type:"button",onClick:b,"aria-label":"Close",className:"".concat(o,"-close")},g||i.createElement("span",{className:"".concat(o,"-close-x")})));var F=i.createElement("div",{className:m()("".concat(o,"-content"),null==v?void 0:v.content),style:null==k?void 0:k.content},r,a,i.createElement("div",(0,w.Z)({className:m()("".concat(o,"-body"),null==v?void 0:v.body),style:(0,x.Z)((0,x.Z)({},E),null==k?void 0:k.body)},h),f),n);return i.createElement("div",{key:"dialog-element",role:"dialog","aria-labelledby":c?d:null,"aria-modal":"true",ref:L,style:(0,x.Z)((0,x.Z)({},l),M),className:m()(o,s),onMouseDown:y,onMouseUp:T},i.createElement("div",{tabIndex:0,ref:D,style:H,"aria-hidden":"true"}),i.createElement($,{shouldUpdate:R||I},S?S(F):F),i.createElement("div",{tabIndex:0,ref:P,style:H,"aria-hidden":"true"}))}),j=i.forwardRef(function(e,t){var n=e.prefixCls,a=e.title,r=e.style,o=e.className,s=e.visible,l=e.forceRender,c=e.destroyOnClose,d=e.motionName,u=e.ariaId,p=e.onVisibleChanged,g=e.mousePosition,b=(0,i.useRef)(),f=i.useState(),E=(0,k.Z)(f,2),h=E[0],S=E[1],y={};function T(){var e,t,n,a,r,i=(n={left:(t=(e=b.current).getBoundingClientRect()).left,top:t.top},r=(a=e.ownerDocument).defaultView||a.parentWindow,n.left+=U(r),n.top+=U(r,!0),n);S(g?"".concat(g.x-i.left,"px ").concat(g.y-i.top,"px"):"")}return h&&(y.transformOrigin=h),i.createElement(B.ZP,{visible:s,onVisibleChanged:p,onAppearPrepare:T,onEnterPrepare:T,forceRender:l,motionName:d,removeOnLeave:c,ref:b},function(s,l){var c=s.className,d=s.style;return i.createElement(z,(0,w.Z)({},e,{ref:t,title:a,ariaId:u,prefixCls:n,holderRef:l,style:(0,x.Z)((0,x.Z)((0,x.Z)({},d),r),y),className:m()(o,c)}))})});function V(e){var t=e.prefixCls,n=e.style,a=e.visible,r=e.maskProps,o=e.motionName,s=e.className;return i.createElement(B.ZP,{key:"mask",visible:a,motionName:o,leavedClassName:"".concat(t,"-mask-hidden")},function(e,a){var o=e.className,l=e.style;return i.createElement("div",(0,w.Z)({ref:a,style:(0,x.Z)((0,x.Z)({},l),n),className:m()("".concat(t,"-mask"),o,s)},r))})}function W(e){var t=e.prefixCls,n=void 0===t?"rc-dialog":t,a=e.zIndex,r=e.visible,o=void 0!==r&&r,s=e.keyboard,l=void 0===s||s,c=e.focusTriggerAfterClose,d=void 0===c||c,u=e.wrapStyle,p=e.wrapClassName,g=e.wrapProps,b=e.onClose,f=e.afterOpenChange,E=e.afterClose,h=e.transitionName,S=e.animation,y=e.closable,T=e.mask,A=void 0===T||T,R=e.maskTransitionName,I=e.maskAnimation,N=e.maskClosable,_=e.maskStyle,v=e.maskProps,C=e.rootClassName,O=e.classNames,U=e.styles,B=(0,i.useRef)(),G=(0,i.useRef)(),$=(0,i.useRef)(),H=i.useState(o),z=(0,k.Z)(H,2),W=z[0],q=z[1],Y=(0,D.Z)();function K(e){null==b||b(e)}var Z=(0,i.useRef)(!1),X=(0,i.useRef)(),Q=null;return(void 0===N||N)&&(Q=function(e){Z.current?Z.current=!1:G.current===e.target&&K(e)}),(0,i.useEffect)(function(){o&&(q(!0),(0,L.Z)(G.current,document.activeElement)||(B.current=document.activeElement))},[o]),(0,i.useEffect)(function(){return function(){clearTimeout(X.current)}},[]),i.createElement("div",(0,w.Z)({className:m()("".concat(n,"-root"),C)},(0,M.Z)(e,{data:!0})),i.createElement(V,{prefixCls:n,visible:A&&o,motionName:F(n,R,I),style:(0,x.Z)((0,x.Z)({zIndex:a},_),null==U?void 0:U.mask),maskProps:v,className:null==O?void 0:O.mask}),i.createElement("div",(0,w.Z)({tabIndex:-1,onKeyDown:function(e){if(l&&e.keyCode===P.Z.ESC){e.stopPropagation(),K(e);return}o&&e.keyCode===P.Z.TAB&&$.current.changeActive(!e.shiftKey)},className:m()("".concat(n,"-wrap"),p,null==O?void 0:O.wrapper),ref:G,onClick:Q,style:(0,x.Z)((0,x.Z)((0,x.Z)({zIndex:a},u),null==U?void 0:U.wrapper),{},{display:W?null:"none"})},g),i.createElement(j,(0,w.Z)({},e,{onMouseDown:function(){clearTimeout(X.current),Z.current=!0},onMouseUp:function(){X.current=setTimeout(function(){Z.current=!1})},ref:$,closable:void 0===y||y,ariaId:Y,prefixCls:n,visible:o&&W,onClose:K,onVisibleChanged:function(e){if(e)!function(){if(!(0,L.Z)(G.current,document.activeElement)){var e;null===(e=$.current)||void 0===e||e.focus()}}();else{if(q(!1),A&&B.current&&d){try{B.current.focus({preventScroll:!0})}catch(e){}B.current=null}W&&(null==E||E())}null==f||f(e)},motionName:F(n,h,S)}))))}j.displayName="Content",n(53850);var q=function(e){var t=e.visible,n=e.getContainer,a=e.forceRender,r=e.destroyOnClose,o=void 0!==r&&r,s=e.afterClose,l=e.panelRef,c=i.useState(t),d=(0,k.Z)(c,2),u=d[0],p=d[1],g=i.useMemo(function(){return{panel:l}},[l]);return(i.useEffect(function(){t&&p(!0)},[t]),a||!o||u)?i.createElement(O.Provider,{value:g},i.createElement(C.Z,{open:t||a||u,autoDestroy:!1,getContainer:n,autoLock:t||u},i.createElement(W,(0,w.Z)({},e,{destroyOnClose:o,afterClose:function(){null==s||s(),p(!1)}})))):null};q.displayName="Dialog";var Y=function(e,t,n){let a=arguments.length>3&&void 0!==arguments[3]?arguments[3]:i.createElement(v.Z,null),r=arguments.length>4&&void 0!==arguments[4]&&arguments[4];if("boolean"==typeof e?!e:void 0===t?!r:!1===t||null===t)return[!1,null];let o="boolean"==typeof t||null==t?a:t;return[!0,n?n(o):o]},K=n(22127),Z=n(86718),X=n(47137),Q=n(92801),J=n(48563);function ee(){}let et=i.createContext({add:ee,remove:ee});var en=n(17094),ea=()=>{let{cancelButtonProps:e,cancelTextLocale:t,onCancel:n}=(0,i.useContext)(R);return i.createElement(y.ZP,Object.assign({onClick:n},e),t)},er=()=>{let{confirmLoading:e,okButtonProps:t,okType:n,okTextLocale:a,onOk:r}=(0,i.useContext)(R);return i.createElement(y.ZP,Object.assign({},(0,T.nx)(n),{loading:e,onClick:r},t),a)},ei=n(4678);function eo(e,t){return i.createElement("span",{className:"".concat(e,"-close-x")},t||i.createElement(v.Z,{className:"".concat(e,"-close-icon")}))}let es=e=>{let t;let{okText:n,okType:a="primary",cancelText:o,confirmLoading:s,onOk:l,onCancel:c,okButtonProps:d,cancelButtonProps:u,footer:p}=e,[g]=(0,E.Z)("Modal",(0,ei.A)()),m={confirmLoading:s,okButtonProps:d,cancelButtonProps:u,okTextLocale:n||(null==g?void 0:g.okText),cancelTextLocale:o||(null==g?void 0:g.cancelText),okType:a,onOk:l,onCancel:c},b=i.useMemo(()=>m,(0,r.Z)(Object.values(m)));return"function"==typeof p||void 0===p?(t=i.createElement(i.Fragment,null,i.createElement(ea,null),i.createElement(er,null)),"function"==typeof p&&(t=p(t,{OkBtn:er,CancelBtn:ea})),t=i.createElement(I,{value:b},t)):t=p,i.createElement(en.n,{disabled:!1},t)};var el=n(11303),ec=n(13703),ed=n(58854),eu=n(80316),ep=n(76585),eg=n(8985);function em(e){return{position:e,inset:0}}let eb=e=>{let{componentCls:t,antCls:n}=e;return[{["".concat(t,"-root")]:{["".concat(t).concat(n,"-zoom-enter, ").concat(t).concat(n,"-zoom-appear")]:{transform:"none",opacity:0,animationDuration:e.motionDurationSlow,userSelect:"none"},["".concat(t).concat(n,"-zoom-leave ").concat(t,"-content")]:{pointerEvents:"none"},["".concat(t,"-mask")]:Object.assign(Object.assign({},em("fixed")),{zIndex:e.zIndexPopupBase,height:"100%",backgroundColor:e.colorBgMask,pointerEvents:"none",["".concat(t,"-hidden")]:{display:"none"}}),["".concat(t,"-wrap")]:Object.assign(Object.assign({},em("fixed")),{zIndex:e.zIndexPopupBase,overflow:"auto",outline:0,WebkitOverflowScrolling:"touch",["&:has(".concat(t).concat(n,"-zoom-enter), &:has(").concat(t).concat(n,"-zoom-appear)")]:{pointerEvents:"none"}})}},{["".concat(t,"-root")]:(0,ec.J$)(e)}]},ef=e=>{let{componentCls:t}=e;return[{["".concat(t,"-root")]:{["".concat(t,"-wrap-rtl")]:{direction:"rtl"},["".concat(t,"-centered")]:{textAlign:"center","&::before":{display:"inline-block",width:0,height:"100%",verticalAlign:"middle",content:'""'},[t]:{top:0,display:"inline-block",paddingBottom:0,textAlign:"start",verticalAlign:"middle"}},["@media (max-width: ".concat(e.screenSMMax,"px)")]:{[t]:{maxWidth:"calc(100vw - 16px)",margin:"".concat((0,eg.bf)(e.marginXS)," auto")},["".concat(t,"-centered")]:{[t]:{flex:1}}}}},{[t]:Object.assign(Object.assign({},(0,el.Wf)(e)),{pointerEvents:"none",position:"relative",top:100,width:"auto",maxWidth:"calc(100vw - ".concat((0,eg.bf)(e.calc(e.margin).mul(2).equal()),")"),margin:"0 auto",paddingBottom:e.paddingLG,["".concat(t,"-title")]:{margin:0,color:e.titleColor,fontWeight:e.fontWeightStrong,fontSize:e.titleFontSize,lineHeight:e.titleLineHeight,wordWrap:"break-word"},["".concat(t,"-content")]:{position:"relative",backgroundColor:e.contentBg,backgroundClip:"padding-box",border:0,borderRadius:e.borderRadiusLG,boxShadow:e.boxShadow,pointerEvents:"auto",padding:e.contentPadding},["".concat(t,"-close")]:Object.assign({position:"absolute",top:e.calc(e.modalHeaderHeight).sub(e.modalCloseBtnSize).div(2).equal(),insetInlineEnd:e.calc(e.modalHeaderHeight).sub(e.modalCloseBtnSize).div(2).equal(),zIndex:e.calc(e.zIndexPopupBase).add(10).equal(),padding:0,color:e.modalCloseIconColor,fontWeight:e.fontWeightStrong,lineHeight:1,textDecoration:"none",background:"transparent",borderRadius:e.borderRadiusSM,width:e.modalCloseBtnSize,height:e.modalCloseBtnSize,border:0,outline:0,cursor:"pointer",transition:"color ".concat(e.motionDurationMid,", background-color ").concat(e.motionDurationMid),"&-x":{display:"flex",fontSize:e.fontSizeLG,fontStyle:"normal",lineHeight:"".concat((0,eg.bf)(e.modalCloseBtnSize)),justifyContent:"center",textTransform:"none",textRendering:"auto"},"&:hover":{color:e.modalIconHoverColor,backgroundColor:e.closeBtnHoverBg,textDecoration:"none"},"&:active":{backgroundColor:e.closeBtnActiveBg}},(0,el.Qy)(e)),["".concat(t,"-header")]:{color:e.colorText,background:e.headerBg,borderRadius:"".concat((0,eg.bf)(e.borderRadiusLG)," ").concat((0,eg.bf)(e.borderRadiusLG)," 0 0"),marginBottom:e.headerMarginBottom,padding:e.headerPadding,borderBottom:e.headerBorderBottom},["".concat(t,"-body")]:{fontSize:e.fontSize,lineHeight:e.lineHeight,wordWrap:"break-word",padding:e.bodyPadding},["".concat(t,"-footer")]:{textAlign:"end",background:e.footerBg,marginTop:e.footerMarginTop,padding:e.footerPadding,borderTop:e.footerBorderTop,borderRadius:e.footerBorderRadius,["> ".concat(e.antCls,"-btn + ").concat(e.antCls,"-btn")]:{marginInlineStart:e.marginXS}},["".concat(t,"-open")]:{overflow:"hidden"}})},{["".concat(t,"-pure-panel")]:{top:"auto",padding:0,display:"flex",flexDirection:"column",["".concat(t,"-content,\n ").concat(t,"-body,\n ").concat(t,"-confirm-body-wrapper")]:{display:"flex",flexDirection:"column",flex:"auto"},["".concat(t,"-confirm-body")]:{marginBottom:"auto"}}}]},eE=e=>{let{componentCls:t}=e;return{["".concat(t,"-root")]:{["".concat(t,"-wrap-rtl")]:{direction:"rtl",["".concat(t,"-confirm-body")]:{direction:"rtl"}}}}},eh=e=>{let t=e.padding,n=e.fontSizeHeading5,a=e.lineHeightHeading5;return(0,eu.TS)(e,{modalHeaderHeight:e.calc(e.calc(a).mul(n).equal()).add(e.calc(t).mul(2).equal()).equal(),modalFooterBorderColorSplit:e.colorSplit,modalFooterBorderStyle:e.lineType,modalFooterBorderWidth:e.lineWidth,modalIconHoverColor:e.colorIconHover,modalCloseIconColor:e.colorIcon,modalCloseBtnSize:e.fontHeight,modalConfirmIconSize:e.fontHeight,modalTitleHeight:e.calc(e.titleFontSize).mul(e.titleLineHeight).equal()})},eS=e=>({footerBg:"transparent",headerBg:e.colorBgElevated,titleLineHeight:e.lineHeightHeading5,titleFontSize:e.fontSizeHeading5,contentBg:e.colorBgElevated,titleColor:e.colorTextHeading,closeBtnHoverBg:e.wireframe?"transparent":e.colorFillContent,closeBtnActiveBg:e.wireframe?"transparent":e.colorFillContentHover,contentPadding:e.wireframe?0:"".concat((0,eg.bf)(e.paddingMD)," ").concat((0,eg.bf)(e.paddingContentHorizontalLG)),headerPadding:e.wireframe?"".concat((0,eg.bf)(e.padding)," ").concat((0,eg.bf)(e.paddingLG)):0,headerBorderBottom:e.wireframe?"".concat((0,eg.bf)(e.lineWidth)," ").concat(e.lineType," ").concat(e.colorSplit):"none",headerMarginBottom:e.wireframe?0:e.marginXS,bodyPadding:e.wireframe?e.paddingLG:0,footerPadding:e.wireframe?"".concat((0,eg.bf)(e.paddingXS)," ").concat((0,eg.bf)(e.padding)):0,footerBorderTop:e.wireframe?"".concat((0,eg.bf)(e.lineWidth)," ").concat(e.lineType," ").concat(e.colorSplit):"none",footerBorderRadius:e.wireframe?"0 0 ".concat((0,eg.bf)(e.borderRadiusLG)," ").concat((0,eg.bf)(e.borderRadiusLG)):0,footerMarginTop:e.wireframe?0:e.marginSM,confirmBodyPadding:e.wireframe?"".concat((0,eg.bf)(2*e.padding)," ").concat((0,eg.bf)(2*e.padding)," ").concat((0,eg.bf)(e.paddingLG)):0,confirmIconMarginInlineEnd:e.wireframe?e.margin:e.marginSM,confirmBtnsMarginTop:e.wireframe?e.marginLG:e.marginSM});var ey=(0,ep.I$)("Modal",e=>{let t=eh(e);return[ef(t),eE(t),eb(t),(0,ed._y)(t,"zoom")]},eS,{unitless:{titleLineHeight:!0}}),eT=n(92935),eA=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n};(0,K.Z)()&&window.document.documentElement&&document.documentElement.addEventListener("click",e=>{a={x:e.pageX,y:e.pageY},setTimeout(()=>{a=null},100)},!0);var eR=e=>{var t;let{getPopupContainer:n,getPrefixCls:r,direction:o,modal:l}=i.useContext(s.E_),c=t=>{let{onCancel:n}=e;null==n||n(t)},{prefixCls:d,className:u,rootClassName:p,open:g,wrapClassName:E,centered:h,getContainer:S,closeIcon:y,closable:T,focusTriggerAfterClose:A=!0,style:R,visible:I,width:N=520,footer:_,classNames:w,styles:k}=e,C=eA(e,["prefixCls","className","rootClassName","open","wrapClassName","centered","getContainer","closeIcon","closable","focusTriggerAfterClose","style","visible","width","footer","classNames","styles"]),O=r("modal",d),x=r(),L=(0,eT.Z)(O),[D,P,M]=ey(O,L),F=m()(E,{["".concat(O,"-centered")]:!!h,["".concat(O,"-wrap-rtl")]:"rtl"===o}),U=null!==_&&i.createElement(es,Object.assign({},e,{onOk:t=>{let{onOk:n}=e;null==n||n(t)},onCancel:c})),[B,G]=Y(T,y,e=>eo(O,e),i.createElement(v.Z,{className:"".concat(O,"-close-icon")}),!0),$=function(e){let t=i.useContext(et),n=i.useRef();return(0,J.zX)(a=>{if(a){let r=e?a.querySelector(e):a;t.add(r),n.current=r}else t.remove(n.current)})}(".".concat(O,"-content")),[H,z]=(0,b.Cn)("Modal",C.zIndex);return D(i.createElement(Q.BR,null,i.createElement(X.Ux,{status:!0,override:!0},i.createElement(Z.Z.Provider,{value:z},i.createElement(q,Object.assign({width:N},C,{zIndex:H,getContainer:void 0===S?n:S,prefixCls:O,rootClassName:m()(P,p,M,L),footer:U,visible:null!=g?g:I,mousePosition:null!==(t=C.mousePosition)&&void 0!==t?t:a,onClose:c,closable:B,closeIcon:G,focusTriggerAfterClose:A,transitionName:(0,f.m)(x,"zoom",e.transitionName),maskTransitionName:(0,f.m)(x,"fade",e.maskTransitionName),className:m()(P,u,null==l?void 0:l.className),style:Object.assign(Object.assign({},null==l?void 0:l.style),R),classNames:Object.assign(Object.assign({wrapper:F},null==l?void 0:l.classNames),w),styles:Object.assign(Object.assign({},null==l?void 0:l.styles),k),panelRef:$}))))))};let eI=e=>{let{componentCls:t,titleFontSize:n,titleLineHeight:a,modalConfirmIconSize:r,fontSize:i,lineHeight:o,modalTitleHeight:s,fontHeight:l,confirmBodyPadding:c}=e,d="".concat(t,"-confirm");return{[d]:{"&-rtl":{direction:"rtl"},["".concat(e.antCls,"-modal-header")]:{display:"none"},["".concat(d,"-body-wrapper")]:Object.assign({},(0,el.dF)()),["&".concat(t," ").concat(t,"-body")]:{padding:c},["".concat(d,"-body")]:{display:"flex",flexWrap:"nowrap",alignItems:"start",["> ".concat(e.iconCls)]:{flex:"none",fontSize:r,marginInlineEnd:e.confirmIconMarginInlineEnd,marginTop:e.calc(e.calc(l).sub(r).equal()).div(2).equal()},["&-has-title > ".concat(e.iconCls)]:{marginTop:e.calc(e.calc(s).sub(r).equal()).div(2).equal()}},["".concat(d,"-paragraph")]:{display:"flex",flexDirection:"column",flex:"auto",rowGap:e.marginXS,maxWidth:"calc(100% - ".concat((0,eg.bf)(e.calc(e.modalConfirmIconSize).add(e.marginSM).equal()),")")},["".concat(d,"-title")]:{color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:n,lineHeight:a},["".concat(d,"-content")]:{color:e.colorText,fontSize:i,lineHeight:o},["".concat(d,"-btns")]:{textAlign:"end",marginTop:e.confirmBtnsMarginTop,["".concat(e.antCls,"-btn + ").concat(e.antCls,"-btn")]:{marginBottom:0,marginInlineStart:e.marginXS}}},["".concat(d,"-error ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorError},["".concat(d,"-warning ").concat(d,"-body > ").concat(e.iconCls,",\n ").concat(d,"-confirm ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorWarning},["".concat(d,"-info ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorInfo},["".concat(d,"-success ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorSuccess}}};var eN=(0,ep.bk)(["Modal","confirm"],e=>[eI(eh(e))],eS,{order:-1e3}),e_=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n};function ev(e){let{prefixCls:t,icon:n,okText:a,cancelText:o,confirmPrefixCls:s,type:l,okCancel:g,footer:b,locale:f}=e,h=e_(e,["prefixCls","icon","okText","cancelText","confirmPrefixCls","type","okCancel","footer","locale"]),S=n;if(!n&&null!==n)switch(l){case"info":S=i.createElement(p.Z,null);break;case"success":S=i.createElement(c.Z,null);break;case"error":S=i.createElement(d.Z,null);break;default:S=i.createElement(u.Z,null)}let y=null!=g?g:"confirm"===l,T=null!==e.autoFocusButton&&(e.autoFocusButton||"ok"),[A]=(0,E.Z)("Modal"),R=f||A,v=a||(y?null==R?void 0:R.okText:null==R?void 0:R.justOkText),w=Object.assign({autoFocusButton:T,cancelTextLocale:o||(null==R?void 0:R.cancelText),okTextLocale:v,mergedOkCancel:y},h),k=i.useMemo(()=>w,(0,r.Z)(Object.values(w))),C=i.createElement(i.Fragment,null,i.createElement(N,null),i.createElement(_,null)),O=void 0!==e.title&&null!==e.title,x="".concat(s,"-body");return i.createElement("div",{className:"".concat(s,"-body-wrapper")},i.createElement("div",{className:m()(x,{["".concat(x,"-has-title")]:O})},S,i.createElement("div",{className:"".concat(s,"-paragraph")},O&&i.createElement("span",{className:"".concat(s,"-title")},e.title),i.createElement("div",{className:"".concat(s,"-content")},e.content))),void 0===b||"function"==typeof b?i.createElement(I,{value:k},i.createElement("div",{className:"".concat(s,"-btns")},"function"==typeof b?b(C,{OkBtn:_,CancelBtn:N}):C)):b,i.createElement(eN,{prefixCls:t}))}let ew=e=>{let{close:t,zIndex:n,afterClose:a,open:r,keyboard:o,centered:s,getContainer:l,maskStyle:c,direction:d,prefixCls:u,wrapClassName:p,rootPrefixCls:g,bodyStyle:E,closable:S=!1,closeIcon:y,modalRender:T,focusTriggerAfterClose:A,onConfirm:R,styles:I}=e,N="".concat(u,"-confirm"),_=e.width||416,v=e.style||{},w=void 0===e.mask||e.mask,k=void 0!==e.maskClosable&&e.maskClosable,C=m()(N,"".concat(N,"-").concat(e.type),{["".concat(N,"-rtl")]:"rtl"===d},e.className),[,O]=(0,h.ZP)(),x=i.useMemo(()=>void 0!==n?n:O.zIndexPopupBase+b.u6,[n,O]);return i.createElement(eR,{prefixCls:u,className:C,wrapClassName:m()({["".concat(N,"-centered")]:!!e.centered},p),onCancel:()=>{null==t||t({triggerCancel:!0}),null==R||R(!1)},open:r,title:"",footer:null,transitionName:(0,f.m)(g||"","zoom",e.transitionName),maskTransitionName:(0,f.m)(g||"","fade",e.maskTransitionName),mask:w,maskClosable:k,style:v,styles:Object.assign({body:E,mask:c},I),width:_,zIndex:x,afterClose:a,keyboard:o,centered:s,getContainer:l,closable:S,closeIcon:y,modalRender:T,focusTriggerAfterClose:A},i.createElement(ev,Object.assign({},e,{confirmPrefixCls:N})))};var ek=e=>{let{rootPrefixCls:t,iconPrefixCls:n,direction:a,theme:r}=e;return i.createElement(l.ZP,{prefixCls:t,iconPrefixCls:n,direction:a,theme:r},i.createElement(ew,Object.assign({},e)))},eC=[];let eO="",ex=e=>{var t,n;let{prefixCls:a,getContainer:r,direction:o}=e,l=(0,ei.A)(),c=(0,i.useContext)(s.E_),d=eO||c.getPrefixCls(),u=a||"".concat(d,"-modal"),p=r;return!1===p&&(p=void 0),i.createElement(ek,Object.assign({},e,{rootPrefixCls:d,prefixCls:u,iconPrefixCls:c.iconPrefixCls,theme:c.theme,direction:null!=o?o:c.direction,locale:null!==(n=null===(t=c.locale)||void 0===t?void 0:t.Modal)&&void 0!==n?n:l,getContainer:p}))};function eL(e){let t;let n=(0,l.w6)(),a=document.createDocumentFragment(),s=Object.assign(Object.assign({},e),{close:u,open:!0});function c(){for(var t=arguments.length,n=Array(t),i=0;ie&&e.triggerCancel);e.onCancel&&s&&e.onCancel.apply(e,[()=>{}].concat((0,r.Z)(n.slice(1))));for(let e=0;e{let t=n.getPrefixCls(void 0,eO),r=n.getIconPrefixCls(),s=n.getTheme(),c=i.createElement(ex,Object.assign({},e));(0,o.s)(i.createElement(l.ZP,{prefixCls:t,iconPrefixCls:r,theme:s},n.holderRender?n.holderRender(c):c),a)})}function u(){for(var t=arguments.length,n=Array(t),a=0;a{"function"==typeof e.afterClose&&e.afterClose(),c.apply(this,n)}})).visible&&delete s.visible,d(s)}return d(s),eC.push(u),{destroy:u,update:function(e){d(s="function"==typeof e?e(s):Object.assign(Object.assign({},s),e))}}}function eD(e){return Object.assign(Object.assign({},e),{type:"warning"})}function eP(e){return Object.assign(Object.assign({},e),{type:"info"})}function eM(e){return Object.assign(Object.assign({},e),{type:"success"})}function eF(e){return Object.assign(Object.assign({},e),{type:"error"})}function eU(e){return Object.assign(Object.assign({},e),{type:"confirm"})}var eB=n(21467),eG=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n},e$=(0,eB.i)(e=>{let{prefixCls:t,className:n,closeIcon:a,closable:r,type:o,title:l,children:c,footer:d}=e,u=eG(e,["prefixCls","className","closeIcon","closable","type","title","children","footer"]),{getPrefixCls:p}=i.useContext(s.E_),g=p(),b=t||p("modal"),f=(0,eT.Z)(g),[E,h,S]=ey(b,f),y="".concat(b,"-confirm"),T={};return T=o?{closable:null!=r&&r,title:"",footer:"",children:i.createElement(ev,Object.assign({},e,{prefixCls:b,confirmPrefixCls:y,rootPrefixCls:g,content:c}))}:{closable:null==r||r,title:l,footer:null!==d&&i.createElement(es,Object.assign({},e)),children:c},E(i.createElement(z,Object.assign({prefixCls:b,className:m()(h,"".concat(b,"-pure-panel"),o&&y,o&&"".concat(y,"-").concat(o),n,S,f)},u,{closeIcon:eo(b,a),closable:r},T)))}),eH=n(79474),ez=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n},ej=i.forwardRef((e,t)=>{var n,{afterClose:a,config:o}=e,l=ez(e,["afterClose","config"]);let[c,d]=i.useState(!0),[u,p]=i.useState(o),{direction:g,getPrefixCls:m}=i.useContext(s.E_),b=m("modal"),f=m(),h=function(){d(!1);for(var e=arguments.length,t=Array(e),n=0;ne&&e.triggerCancel);u.onCancel&&a&&u.onCancel.apply(u,[()=>{}].concat((0,r.Z)(t.slice(1))))};i.useImperativeHandle(t,()=>({destroy:h,update:e=>{p(t=>Object.assign(Object.assign({},t),e))}}));let S=null!==(n=u.okCancel)&&void 0!==n?n:"confirm"===u.type,[y]=(0,E.Z)("Modal",eH.Z.Modal);return i.createElement(ek,Object.assign({prefixCls:b,rootPrefixCls:f},u,{close:h,open:c,afterClose:()=>{var e;a(),null===(e=u.afterClose)||void 0===e||e.call(u)},okText:u.okText||(S?null==y?void 0:y.okText:null==y?void 0:y.justOkText),direction:u.direction||g,cancelText:u.cancelText||(null==y?void 0:y.cancelText)},l))});let eV=0,eW=i.memo(i.forwardRef((e,t)=>{let[n,a]=function(){let[e,t]=i.useState([]);return[e,i.useCallback(e=>(t(t=>[].concat((0,r.Z)(t),[e])),()=>{t(t=>t.filter(t=>t!==e))}),[])]}();return i.useImperativeHandle(t,()=>({patchElement:a}),[]),i.createElement(i.Fragment,null,n)}));function eq(e){return eL(eD(e))}eR.useModal=function(){let e=i.useRef(null),[t,n]=i.useState([]);i.useEffect(()=>{t.length&&((0,r.Z)(t).forEach(e=>{e()}),n([]))},[t]);let a=i.useCallback(t=>function(a){var o;let s,l;eV+=1;let c=i.createRef(),d=new Promise(e=>{s=e}),u=!1,p=i.createElement(ej,{key:"modal-".concat(eV),config:t(a),ref:c,afterClose:()=>{null==l||l()},isSilent:()=>u,onConfirm:e=>{s(e)}});return(l=null===(o=e.current)||void 0===o?void 0:o.patchElement(p))&&eC.push(l),{destroy:()=>{function e(){var e;null===(e=c.current)||void 0===e||e.destroy()}c.current?e():n(t=>[].concat((0,r.Z)(t),[e]))},update:e=>{function t(){var t;null===(t=c.current)||void 0===t||t.update(e)}c.current?t():n(e=>[].concat((0,r.Z)(e),[t]))},then:e=>(u=!0,d.then(e))}},[]);return[i.useMemo(()=>({info:a(eP),success:a(eM),error:a(eF),warning:a(eD),confirm:a(eU)}),[]),i.createElement(eW,{key:"modal-holder",ref:e})]},eR.info=function(e){return eL(eP(e))},eR.success=function(e){return eL(eM(e))},eR.error=function(e){return eL(eF(e))},eR.warning=eq,eR.warn=eq,eR.confirm=function(e){return eL(eU(e))},eR.destroyAll=function(){for(;eC.length;){let e=eC.pop();e&&e()}},eR.config=function(e){let{rootPrefixCls:t}=e;eO=t},eR._InternalPanelDoNotUseOrYouWillBeFired=e$;var eY=eR},13703:function(e,t,n){n.d(t,{J$:function(){return s}});var a=n(8985),r=n(59353);let i=new a.E4("antFadeIn",{"0%":{opacity:0},"100%":{opacity:1}}),o=new a.E4("antFadeOut",{"0%":{opacity:1},"100%":{opacity:0}}),s=function(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],{antCls:n}=e,a="".concat(n,"-fade"),s=t?"&":"";return[(0,r.R)(a,i,o,e.motionDurationMid,t),{["\n ".concat(s).concat(a,"-enter,\n ").concat(s).concat(a,"-appear\n ")]:{opacity:0,animationTimingFunction:"linear"},["".concat(s).concat(a,"-leave")]:{animationTimingFunction:"linear"}}]}},44056:function(e){e.exports=function(e,n){for(var a,r,i,o=e||"",s=n||"div",l={},c=0;c4&&m.slice(0,4)===o&&s.test(t)&&("-"===t.charAt(4)?b=o+(n=t.slice(5).replace(l,u)).charAt(0).toUpperCase()+n.slice(1):(g=(p=t).slice(4),t=l.test(g)?p:("-"!==(g=g.replace(c,d)).charAt(0)&&(g="-"+g),o+g)),f=r),new f(b,t))};var s=/^data[-\w.:]+$/i,l=/-[a-z]/g,c=/[A-Z]/g;function d(e){return"-"+e.toLowerCase()}function u(e){return e.charAt(1).toUpperCase()}},31872:function(e,t,n){var a=n(96130),r=n(64730),i=n(61861),o=n(46982),s=n(83671),l=n(53618);e.exports=a([i,r,o,s,l])},83671:function(e,t,n){var a=n(7667),r=n(13585),i=a.booleanish,o=a.number,s=a.spaceSeparated;e.exports=r({transform:function(e,t){return"role"===t?t:"aria-"+t.slice(4).toLowerCase()},properties:{ariaActiveDescendant:null,ariaAtomic:i,ariaAutoComplete:null,ariaBusy:i,ariaChecked:i,ariaColCount:o,ariaColIndex:o,ariaColSpan:o,ariaControls:s,ariaCurrent:null,ariaDescribedBy:s,ariaDetails:null,ariaDisabled:i,ariaDropEffect:s,ariaErrorMessage:null,ariaExpanded:i,ariaFlowTo:s,ariaGrabbed:i,ariaHasPopup:null,ariaHidden:i,ariaInvalid:null,ariaKeyShortcuts:null,ariaLabel:null,ariaLabelledBy:s,ariaLevel:o,ariaLive:null,ariaModal:i,ariaMultiLine:i,ariaMultiSelectable:i,ariaOrientation:null,ariaOwns:s,ariaPlaceholder:null,ariaPosInSet:o,ariaPressed:i,ariaReadOnly:i,ariaRelevant:null,ariaRequired:i,ariaRoleDescription:s,ariaRowCount:o,ariaRowIndex:o,ariaRowSpan:o,ariaSelected:i,ariaSetSize:o,ariaSort:null,ariaValueMax:o,ariaValueMin:o,ariaValueNow:o,ariaValueText:null,role:null}})},53618:function(e,t,n){var a=n(7667),r=n(13585),i=n(46640),o=a.boolean,s=a.overloadedBoolean,l=a.booleanish,c=a.number,d=a.spaceSeparated,u=a.commaSeparated;e.exports=r({space:"html",attributes:{acceptcharset:"accept-charset",classname:"class",htmlfor:"for",httpequiv:"http-equiv"},transform:i,mustUseProperty:["checked","multiple","muted","selected"],properties:{abbr:null,accept:u,acceptCharset:d,accessKey:d,action:null,allow:null,allowFullScreen:o,allowPaymentRequest:o,allowUserMedia:o,alt:null,as:null,async:o,autoCapitalize:null,autoComplete:d,autoFocus:o,autoPlay:o,capture:o,charSet:null,checked:o,cite:null,className:d,cols:c,colSpan:null,content:null,contentEditable:l,controls:o,controlsList:d,coords:c|u,crossOrigin:null,data:null,dateTime:null,decoding:null,default:o,defer:o,dir:null,dirName:null,disabled:o,download:s,draggable:l,encType:null,enterKeyHint:null,form:null,formAction:null,formEncType:null,formMethod:null,formNoValidate:o,formTarget:null,headers:d,height:c,hidden:o,high:c,href:null,hrefLang:null,htmlFor:d,httpEquiv:d,id:null,imageSizes:null,imageSrcSet:u,inputMode:null,integrity:null,is:null,isMap:o,itemId:null,itemProp:d,itemRef:d,itemScope:o,itemType:d,kind:null,label:null,lang:null,language:null,list:null,loading:null,loop:o,low:c,manifest:null,max:null,maxLength:c,media:null,method:null,min:null,minLength:c,multiple:o,muted:o,name:null,nonce:null,noModule:o,noValidate:o,onAbort:null,onAfterPrint:null,onAuxClick:null,onBeforePrint:null,onBeforeUnload:null,onBlur:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onContextMenu:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnded:null,onError:null,onFocus:null,onFormData:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLanguageChange:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadEnd:null,onLoadStart:null,onMessage:null,onMessageError:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRejectionHandled:null,onReset:null,onResize:null,onScroll:null,onSecurityPolicyViolation:null,onSeeked:null,onSeeking:null,onSelect:null,onSlotChange:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnhandledRejection:null,onUnload:null,onVolumeChange:null,onWaiting:null,onWheel:null,open:o,optimum:c,pattern:null,ping:d,placeholder:null,playsInline:o,poster:null,preload:null,readOnly:o,referrerPolicy:null,rel:d,required:o,reversed:o,rows:c,rowSpan:c,sandbox:d,scope:null,scoped:o,seamless:o,selected:o,shape:null,size:c,sizes:null,slot:null,span:c,spellCheck:l,src:null,srcDoc:null,srcLang:null,srcSet:u,start:c,step:null,style:null,tabIndex:c,target:null,title:null,translate:null,type:null,typeMustMatch:o,useMap:null,value:l,width:c,wrap:null,align:null,aLink:null,archive:d,axis:null,background:null,bgColor:null,border:c,borderColor:null,bottomMargin:c,cellPadding:null,cellSpacing:null,char:null,charOff:null,classId:null,clear:null,code:null,codeBase:null,codeType:null,color:null,compact:o,declare:o,event:null,face:null,frame:null,frameBorder:null,hSpace:c,leftMargin:c,link:null,longDesc:null,lowSrc:null,marginHeight:c,marginWidth:c,noResize:o,noHref:o,noShade:o,noWrap:o,object:null,profile:null,prompt:null,rev:null,rightMargin:c,rules:null,scheme:null,scrolling:l,standby:null,summary:null,text:null,topMargin:c,valueType:null,version:null,vAlign:null,vLink:null,vSpace:c,allowTransparency:null,autoCorrect:null,autoSave:null,disablePictureInPicture:o,disableRemotePlayback:o,prefix:null,property:null,results:c,security:null,unselectable:null}})},46640:function(e,t,n){var a=n(25852);e.exports=function(e,t){return a(e,t.toLowerCase())}},25852:function(e){e.exports=function(e,t){return t in e?e[t]:t}},13585:function(e,t,n){var a=n(39900),r=n(94949),i=n(7478);e.exports=function(e){var t,n,o=e.space,s=e.mustUseProperty||[],l=e.attributes||{},c=e.properties,d=e.transform,u={},p={};for(t in c)n=new i(t,d(l,t),c[t],o),-1!==s.indexOf(t)&&(n.mustUseProperty=!0),u[t]=n,p[a(t)]=t,p[a(n.attribute)]=t;return new r(u,p,o)}},7478:function(e,t,n){var a=n(74108),r=n(7667);e.exports=s,s.prototype=new a,s.prototype.defined=!0;var i=["boolean","booleanish","overloadedBoolean","number","commaSeparated","spaceSeparated","commaOrSpaceSeparated"],o=i.length;function s(e,t,n,s){var l,c,d,u=-1;for(s&&(this.space=s),a.call(this,e,t);++u