From 1890fde3f377b0896e7ca5908953ef948ec1c8a7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 07:02:12 -0800 Subject: [PATCH 001/113] (Proxy) add support for DOCS_URL and REDOC_URL (#6806) * add support for DOCS_URL and REDOC_URL * document env vars * add unit tests for docs url and redocs url --- docs/my-website/docs/proxy/configs.md | 2 + litellm/proxy/proxy_server.py | 7 ++-- litellm/proxy/utils.py | 29 ++++++++++++++ tests/proxy_unit_tests/test_proxy_utils.py | 44 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 1609e16ae..3b6b336d6 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -934,6 +934,7 @@ router_settings: | DOCS_DESCRIPTION | Description text for documentation pages | DOCS_FILTERED | Flag indicating filtered documentation | DOCS_TITLE | Title of the documentation pages +| DOCS_URL | The path to the Swagger API documentation. **By default this is "/"** | EMAIL_SUPPORT_CONTACT | Support contact email address | GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket | GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file @@ -1041,6 +1042,7 @@ router_settings: | REDIS_HOST | Hostname for Redis server | REDIS_PASSWORD | Password for Redis service | REDIS_PORT | Port number for Redis server +| REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"** | SERVER_ROOT_PATH | Root path for the server application | SET_VERBOSE | Flag to enable verbose logging | SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2ece9705a..4d4c6a1a2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -222,7 +222,9 @@ from litellm.proxy.utils import ( PrismaClient, ProxyLogging, _cache_user_row, + _get_docs_url, _get_projected_spend_over_limit, + _get_redoc_url, _is_projected_spend_over_limit, _is_valid_team_configs, get_error_message_str, @@ -344,7 +346,6 @@ ui_message += "\n\n💸 [```LiteLLM Model Cost Map```](https://models.litellm.ai custom_swagger_message = "[**Customize Swagger Docs**](https://docs.litellm.ai/docs/proxy/enterprise#swagger-docs---custom-routes--branding)" ### CUSTOM BRANDING [ENTERPRISE FEATURE] ### -_docs_url = None if os.getenv("NO_DOCS", "False") == "True" else "/" _title = os.getenv("DOCS_TITLE", "LiteLLM API") if premium_user else "LiteLLM API" _description = ( os.getenv( @@ -355,9 +356,9 @@ _description = ( else f"Proxy Server to call 100+ LLMs in the OpenAI format. {custom_swagger_message}\n\n{ui_message}" ) - app = FastAPI( - docs_url=_docs_url, + docs_url=_get_docs_url(), + redoc_url=_get_redoc_url(), title=_title, description=_description, version=version, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bcd3ce16c..e495f3490 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3099,6 +3099,34 @@ def get_error_message_str(e: Exception) -> str: return error_message +def _get_redoc_url() -> str: + """ + Get the redoc URL from the environment variables. + + - If REDOC_URL is set, return it. + - Otherwise, default to "/redoc". + """ + return os.getenv("REDOC_URL", "/redoc") + + +def _get_docs_url() -> Optional[str]: + """ + Get the docs URL from the environment variables. + + - If DOCS_URL is set, return it. + - If NO_DOCS is True, return None. + - Otherwise, default to "/". + """ + docs_url = os.getenv("DOCS_URL", None) + if docs_url: + return docs_url + + if os.getenv("NO_DOCS", "False") == "True": + return None + + # default to "/" + return "/" + def handle_exception_on_proxy(e: Exception) -> ProxyException: """ Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible @@ -3120,3 +3148,4 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException: param=getattr(e, "param", "None"), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 2e857808d..607e54225 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2,6 +2,7 @@ import asyncio import os import sys from unittest.mock import Mock +from litellm.proxy.utils import _get_redoc_url, _get_docs_url import pytest from fastapi import Request @@ -530,3 +531,46 @@ def test_prepare_key_update_data(): data = UpdateKeyRequest(key="test_key", metadata=None) updated_data = prepare_key_update_data(data, existing_key_row) assert updated_data["metadata"] == None + + +@pytest.mark.parametrize( + "env_value, expected_url", + [ + (None, "/redoc"), # default case + ("/custom-redoc", "/custom-redoc"), # custom URL + ("https://example.com/redoc", "https://example.com/redoc"), # full URL + ], +) +def test_get_redoc_url(env_value, expected_url): + if env_value is not None: + os.environ["REDOC_URL"] = env_value + else: + os.environ.pop("REDOC_URL", None) # ensure env var is not set + + result = _get_redoc_url() + assert result == expected_url + + +@pytest.mark.parametrize( + "env_vars, expected_url", + [ + ({}, "/"), # default case + ({"DOCS_URL": "/custom-docs"}, "/custom-docs"), # custom URL + ( + {"DOCS_URL": "https://example.com/docs"}, + "https://example.com/docs", + ), # full URL + ({"NO_DOCS": "True"}, None), # docs disabled + ], +) +def test_get_docs_url(env_vars, expected_url): + # Clear relevant environment variables + for key in ["DOCS_URL", "NO_DOCS"]: + os.environ.pop(key, None) + + # Set test environment variables + for key, value in env_vars.items(): + os.environ[key] = value + + result = _get_docs_url() + assert result == expected_url From f4ec93fbc333a47ed48caf85b75a9ed94b9ffd94 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 19 Nov 2024 22:04:39 +0530 Subject: [PATCH 002/113] fix(anthropic/chat/transformation.py): add json schema as values: json_schema fixes passing pydantic obj to anthropic Fixes https://github.com/BerriAI/litellm/issues/6766 --- litellm/llms/anthropic/chat/transformation.py | 2 +- ...odel_prices_and_context_window_backup.json | 34 ++++++--------- model_prices_and_context_window.json | 34 ++++++--------- tests/llm_translation/base_llm_unit_tests.py | 43 ++++++++++++++++--- 4 files changed, 63 insertions(+), 50 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1419d7ef2..ec981096c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -374,7 +374,7 @@ class AnthropicConfig: _input_schema["additionalProperties"] = True _input_schema["properties"] = {} else: - _input_schema["properties"] = json_schema + _input_schema["properties"] = {"values": json_schema} _tool = AnthropicMessagesTool(name="json_tool_call", input_schema=_input_schema) return _tool diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 827434123..4a8e9e32a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1884,7 +1884,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-haiku-20241022": { "max_tokens": 8192, @@ -1900,7 +1901,8 @@ "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, "supports_prompt_caching": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "supports_response_schema": true }, "claude-3-opus-20240229": { "max_tokens": 4096, @@ -1916,7 +1918,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 395, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-sonnet-20240229": { "max_tokens": 4096, @@ -1930,7 +1933,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20240620": { "max_tokens": 8192, @@ -1946,7 +1950,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20241022": { "max_tokens": 8192, @@ -1962,7 +1967,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "text-bison": { "max_tokens": 2048, @@ -3852,22 +3858,6 @@ "supports_function_calling": true, "tool_use_system_prompt_tokens": 264 }, - "anthropic/claude-3-5-sonnet-20241022": { - "max_tokens": 8192, - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_creation_input_token_cost": 0.00000375, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "anthropic", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, - "supports_prompt_caching": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "max_tokens": 8192, "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 827434123..4a8e9e32a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1884,7 +1884,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-haiku-20241022": { "max_tokens": 8192, @@ -1900,7 +1901,8 @@ "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, "supports_prompt_caching": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "supports_response_schema": true }, "claude-3-opus-20240229": { "max_tokens": 4096, @@ -1916,7 +1918,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 395, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-sonnet-20240229": { "max_tokens": 4096, @@ -1930,7 +1933,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20240620": { "max_tokens": 8192, @@ -1946,7 +1950,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20241022": { "max_tokens": 8192, @@ -1962,7 +1967,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "text-bison": { "max_tokens": 2048, @@ -3852,22 +3858,6 @@ "supports_function_calling": true, "tool_use_system_prompt_tokens": 264 }, - "anthropic/claude-3-5-sonnet-20241022": { - "max_tokens": 8192, - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_creation_input_token_cost": 0.00000375, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "anthropic", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, - "supports_prompt_caching": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "max_tokens": 8192, "max_input_tokens": 200000, diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 955eed957..74fff60a4 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -42,11 +42,14 @@ class BaseLLMChatTest(ABC): "content": [{"type": "text", "text": "Hello, how are you?"}], } ] - response = litellm.completion( - **base_completion_call_args, - messages=messages, - ) - assert response is not None + try: + response = litellm.completion( + **base_completion_call_args, + messages=messages, + ) + assert response is not None + except litellm.InternalServerError: + pass # for OpenAI the content contains the JSON schema, so we need to assert that the content is not None assert response.choices[0].message.content is not None @@ -89,6 +92,36 @@ class BaseLLMChatTest(ABC): # relevant issue: https://github.com/BerriAI/litellm/issues/6741 assert response.choices[0].message.content is not None + def test_json_response_pydantic_obj(self): + from pydantic import BaseModel + from litellm.utils import supports_response_schema + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + class TestModel(BaseModel): + first_response: str + + base_completion_call_args = self.get_base_completion_call_args() + if not supports_response_schema(base_completion_call_args["model"], None): + pytest.skip("Model does not support response schema") + + try: + res = litellm.completion( + **base_completion_call_args, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + { + "role": "user", + "content": "What is the capital of France?", + }, + ], + response_format=TestModel, + ) + assert res is not None + except litellm.InternalServerError: + pytest.skip("Model is overloaded") + def test_json_response_format_stream(self): """ Test that the JSON response format with streaming is supported by the LLM API From 98c78890137ba18c93b70d2d9e8a09dcf3c5fab9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 14:50:51 -0800 Subject: [PATCH 003/113] feat - add qwen2p5-coder-32b-instruct (#6818) --- litellm/model_prices_and_context_window_backup.json | 11 +++++++++++ model_prices_and_context_window.json | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 827434123..b665f4381 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6489,6 +6489,17 @@ "supports_function_calling": true, "source": "https://fireworks.ai/pricing" }, + "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "source": "https://fireworks.ai/pricing" + }, "fireworks_ai/accounts/fireworks/models/yi-large": { "max_tokens": 32768, "max_input_tokens": 32768, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 827434123..b665f4381 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6489,6 +6489,17 @@ "supports_function_calling": true, "source": "https://fireworks.ai/pricing" }, + "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct": { + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "source": "https://fireworks.ai/pricing" + }, "fireworks_ai/accounts/fireworks/models/yi-large": { "max_tokens": 32768, "max_input_tokens": 32768, From e89dcccdd9f8cf95b0519385c07e540b14c864ab Mon Sep 17 00:00:00 2001 From: Show <35062952+BrunooShow@users.noreply.github.com> Date: Wed, 20 Nov 2024 00:04:33 +0100 Subject: [PATCH 004/113] (feat): Add timestamp_granularities parameter to transcription API (#6457) * Add timestamp_granularities parameter to transcription API * add param to the local test --- litellm/main.py | 2 ++ litellm/utils.py | 1 + tests/local_testing/test_whisper.py | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 3b4a99413..0afce3db5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4729,6 +4729,7 @@ def transcription( response_format: Optional[ Literal["json", "text", "srt", "verbose_json", "vtt"] ] = None, + timestamp_granularities: List[Literal["word", "segment"]] = None, temperature: Optional[int] = None, # openai defaults this to 0 ## LITELLM PARAMS ## user: Optional[str] = None, @@ -4778,6 +4779,7 @@ def transcription( language=language, prompt=prompt, response_format=response_format, + timestamp_granularities=timestamp_granularities, temperature=temperature, custom_llm_provider=custom_llm_provider, drop_params=drop_params, diff --git a/litellm/utils.py b/litellm/utils.py index cb8a53354..2dce9db89 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2125,6 +2125,7 @@ def get_optional_params_transcription( prompt: Optional[str] = None, response_format: Optional[str] = None, temperature: Optional[int] = None, + timestamp_granularities: Optional[List[Literal["word", "segment"]]] = None, custom_llm_provider: Optional[str] = None, drop_params: Optional[bool] = None, **kwargs, diff --git a/tests/local_testing/test_whisper.py b/tests/local_testing/test_whisper.py index f66ad8b13..7d5d0d710 100644 --- a/tests/local_testing/test_whisper.py +++ b/tests/local_testing/test_whisper.py @@ -53,8 +53,9 @@ from litellm import Router ) @pytest.mark.parametrize("response_format", ["json", "vtt"]) @pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("timestamp_granularities", [["word"], ["segment"]]) @pytest.mark.asyncio -async def test_transcription(model, api_key, api_base, response_format, sync_mode): +async def test_transcription(model, api_key, api_base, response_format, sync_mode, timestamp_granularities): if sync_mode: transcript = litellm.transcription( model=model, @@ -62,6 +63,7 @@ async def test_transcription(model, api_key, api_base, response_format, sync_mod api_key=api_key, api_base=api_base, response_format=response_format, + timestamp_granularities=timestamp_granularities, drop_params=True, ) else: From 07ba5379709f3a9f6e46ee7015c3b57d3a140bbc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 04:56:18 +0530 Subject: [PATCH 005/113] fix(databricks/chat.py): handle max_retries optional param handling for openai-like calls Fixes issue with calling finetuned vertex ai models via databricks route --- litellm/llms/databricks/chat.py | 3 +++ tests/llm_translation/test_optional_params.py | 12 +++++++++++- .../local_testing/test_amazing_vertex_completion.py | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/litellm/llms/databricks/chat.py b/litellm/llms/databricks/chat.py index eb0cb341e..79e885646 100644 --- a/litellm/llms/databricks/chat.py +++ b/litellm/llms/databricks/chat.py @@ -470,6 +470,9 @@ class DatabricksChatCompletion(BaseLLM): optional_params[k] = v stream: bool = optional_params.get("stream", None) or False + optional_params.pop( + "max_retries", None + ) # [TODO] add max retry support at llm api call level optional_params["stream"] = stream data = { diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 029e91513..33e708ff4 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -923,7 +923,6 @@ def test_watsonx_text_top_k(): assert optional_params["top_k"] == 10 - def test_together_ai_model_params(): optional_params = get_optional_params( model="together_ai", custom_llm_provider="together_ai", logprobs=1 @@ -931,6 +930,7 @@ def test_together_ai_model_params(): print(optional_params) assert optional_params["logprobs"] == 1 + def test_forward_user_param(): from litellm.utils import get_supported_openai_params, get_optional_params @@ -943,6 +943,7 @@ def test_forward_user_param(): assert optional_params["metadata"]["user_id"] == "test_user" + def test_lm_studio_embedding_params(): optional_params = get_optional_params_embeddings( model="lm_studio/gemma2-9b-it", @@ -951,3 +952,12 @@ def test_lm_studio_embedding_params(): drop_params=True, ) assert len(optional_params) == 0 + + +def test_vertex_ft_models_optional_params(): + optional_params = get_optional_params( + model="meta-llama/Llama-3.1-8B-Instruct", + custom_llm_provider="vertex_ai", + max_retries=3, + ) + assert "max_retries" not in optional_params diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 3bf36dda8..f801a53ce 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -3129,9 +3129,12 @@ async def test_vertexai_embedding_finetuned(respx_mock: MockRouter): assert all(isinstance(x, float) for x in embedding["embedding"]) +@pytest.mark.parametrize("max_retries", [None, 3]) @pytest.mark.asyncio @pytest.mark.respx -async def test_vertexai_model_garden_model_completion(respx_mock: MockRouter): +async def test_vertexai_model_garden_model_completion( + respx_mock: MockRouter, max_retries +): """ Relevant issue: https://github.com/BerriAI/litellm/issues/6480 @@ -3189,6 +3192,7 @@ async def test_vertexai_model_garden_model_completion(respx_mock: MockRouter): messages=messages, vertex_project="633608382793", vertex_location="us-central1", + max_retries=max_retries, ) # Assert request was made correctly From cf579fe644055c0011413ed6d902915920844a4b Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 20 Nov 2024 05:03:42 +0530 Subject: [PATCH 006/113] Litellm stable pr 10 30 2024 (#6821) * Update organization_endpoints.py to be able to list organizations (#6473) * Update organization_endpoints.py to be able to list organizations * Update test_organizations.py * Update test_organizations.py add test for list * Update test_organizations.py correct indentation * Add unreleased Claude 3.5 Haiku models. (#6476) --------- Co-authored-by: superpoussin22 Co-authored-by: David Manouchehri --- ...odel_prices_and_context_window_backup.json | 18 ++++++++- .../organization_endpoints.py | 39 +++++++++++++++++++ model_prices_and_context_window.json | 12 ++++++ tests/test_organizations.py | 38 ++++++++++++++++++ 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b665f4381..f8dd86cbc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2803,6 +2803,18 @@ "supports_vision": true, "supports_assistant_prefill": true }, + "vertex_ai/claude-3-5-haiku@20241022": { + "max_tokens": 8192, + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_assistant_prefill": true + }, "vertex_ai/claude-3-haiku@20240307": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -4662,7 +4674,8 @@ "litellm_provider": "bedrock", "mode": "chat", "supports_assistant_prefill": true, - "supports_function_calling": true + "supports_function_calling": true, + "supports_vision": true }, "us.anthropic.claude-3-opus-20240229-v1:0": { "max_tokens": 4096, @@ -4728,7 +4741,8 @@ "output_cost_per_token": 0.000005, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_vision": true }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "max_tokens": 4096, diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index f448d2fad..5f58c4231 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -198,6 +198,45 @@ async def delete_organization(): pass +@router.get( + "/organization/list", + tags=["organization management"], + dependencies=[Depends(user_api_key_auth)], +) +async def list_organization( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + ``` + curl --location --request GET 'http://0.0.0.0:4000/organization/list' \ + --header 'Authorization: Bearer sk-1234' + ``` + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + if ( + user_api_key_dict.user_role is None + or user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN + ): + raise HTTPException( + status_code=401, + detail={ + "error": f"Only admins can list orgs. Your role is = {user_api_key_dict.user_role}" + }, + ) + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + response= await prisma_client.db.litellm_organizationtable.find_many() + + return response + + @router.post( "/organization/info", tags=["organization management"], diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b665f4381..815672ff2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2803,6 +2803,18 @@ "supports_vision": true, "supports_assistant_prefill": true }, + "vertex_ai/claude-3-5-haiku@20241022": { + "max_tokens": 8192, + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "supports_assistant_prefill": true + }, "vertex_ai/claude-3-haiku@20240307": { "max_tokens": 4096, "max_input_tokens": 200000, diff --git a/tests/test_organizations.py b/tests/test_organizations.py index 5d9eb2e27..d62380b4a 100644 --- a/tests/test_organizations.py +++ b/tests/test_organizations.py @@ -29,6 +29,22 @@ async def new_organization(session, i, organization_alias, max_budget=None): return await response.json() +async def list_organization(session, i): + url = "http://0.0.0.0:4000/organization/list" + headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} + + async with session.post(url, headers=headers) as response: + status = response.status + response_json = await response.json() + + print(f"Response {i} (Status code: {status}):") + print(response_json) + print() + + if status != 200: + raise Exception(f"Request {i} did not return a 200 status code: {status}") + + return await response.json() @pytest.mark.asyncio async def test_organization_new(): @@ -44,3 +60,25 @@ async def test_organization_new(): for i in range(1, 20) ] await asyncio.gather(*tasks) + +@pytest.mark.asyncio +async def test_organization_list(): + """ + create 2 new Organizations + check if the Organization list is not empty + """ + organization_alias = f"Organization: {uuid.uuid4()}" + async with aiohttp.ClientSession() as session: + tasks = [ + new_organization( + session=session, i=0, organization_alias=organization_alias + ) + for i in range(1, 2) + ] + await asyncio.gather(*tasks) + + response_json = await list_organization(session) + print(len(response_json)) + + if len(response_json)==0: + raise Exception(f"Return empty list of organization") From 5be647dd76dff326b22220dea8ace9b0233ec1a7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:11:28 +0530 Subject: [PATCH 007/113] build(ui/): add team admins via proxy ui --- ui/litellm-dashboard/src/components/admins.tsx | 7 ------- ui/litellm-dashboard/src/components/teams.tsx | 8 +++++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/admins.tsx b/ui/litellm-dashboard/src/components/admins.tsx index 80c849ac1..f226d1c11 100644 --- a/ui/litellm-dashboard/src/components/admins.tsx +++ b/ui/litellm-dashboard/src/components/admins.tsx @@ -314,13 +314,6 @@ const AdminPanel: React.FC = ({ className="px-3 py-2 border rounded-md w-full" /> - {/*
OR
- - - */}
Add member diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index 90a29de32..11664bd02 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -381,7 +381,7 @@ const Team: React.FC = ({ if (accessToken != null && teams != null) { message.info("Adding Member"); const user_role: Member = { - role: "user", + role: formValues.role, user_email: formValues.user_email, user_id: formValues.user_id, }; @@ -809,6 +809,12 @@ const Team: React.FC = ({ className="px-3 py-2 border rounded-md w-full" /> + + + user + admin + +
Add member From aed1cd3283d9367c86f29536ba6b89966939ff3a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:17:27 +0530 Subject: [PATCH 008/113] fix: fix linting error --- litellm/main.py | 2 +- tests/local_testing/test_whisper.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 0afce3db5..f93eeeda9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4729,7 +4729,7 @@ def transcription( response_format: Optional[ Literal["json", "text", "srt", "verbose_json", "vtt"] ] = None, - timestamp_granularities: List[Literal["word", "segment"]] = None, + timestamp_granularities: Optional[List[Literal["word", "segment"]]] = None, temperature: Optional[int] = None, # openai defaults this to 0 ## LITELLM PARAMS ## user: Optional[str] = None, diff --git a/tests/local_testing/test_whisper.py b/tests/local_testing/test_whisper.py index 7d5d0d710..e2663a3ab 100644 --- a/tests/local_testing/test_whisper.py +++ b/tests/local_testing/test_whisper.py @@ -53,9 +53,11 @@ from litellm import Router ) @pytest.mark.parametrize("response_format", ["json", "vtt"]) @pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.parametrize("timestamp_granularities", [["word"], ["segment"]]) +@pytest.mark.parametrize("timestamp_granularities", [None, ["word"], ["segment"]]) @pytest.mark.asyncio -async def test_transcription(model, api_key, api_base, response_format, sync_mode, timestamp_granularities): +async def test_transcription( + model, api_key, api_base, response_format, sync_mode, timestamp_granularities +): if sync_mode: transcript = litellm.transcription( model=model, From 353558e9787303a12c92fee7ef4227ec4b0d01a5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:19:52 +0530 Subject: [PATCH 009/113] test: fix test --- tests/llm_translation/test_anthropic_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 8a788e0fb..c6181f1ba 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -657,7 +657,7 @@ def test_create_json_tool_call_for_response_format(): _input_schema = tool.get("input_schema") assert _input_schema is not None assert _input_schema.get("type") == "object" - assert _input_schema.get("properties") == custom_schema + assert _input_schema.get("properties") == {"values": custom_schema} assert "additionalProperties" not in _input_schema From 7bc9f1299c2928be0afd92f4ba6eb49acde970e4 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:28:35 +0530 Subject: [PATCH 010/113] docs(vertex.md): refactor docs --- docs/my-website/docs/providers/vertex.md | 181 +++++++++++------------ 1 file changed, 90 insertions(+), 91 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 605762422..a7b363be1 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -572,6 +572,96 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server + +## Authentication - vertex_project, vertex_location, etc. + +Set your vertex credentials via: +- dynamic params +OR +- env vars + + +### **Dynamic Params** + +You can set: +- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json +- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.) +- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials + +as dynamic params for a `litellm.completion` call. + + + + +```python +from litellm import completion +import json + +## GET CREDENTIALS +file_path = 'path/to/vertex_ai_service_account.json' + +# Load the JSON file +with open(file_path, 'r') as file: + vertex_credentials = json.load(file) + +# Convert to JSON string +vertex_credentials_json = json.dumps(vertex_credentials) + + +response = completion( + model="vertex_ai/gemini-pro", + messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}], + vertex_credentials=vertex_credentials_json, + vertex_project="my-special-project", + vertex_location="my-special-location" +) +``` + + + + +```yaml +model_list: + - model_name: gemini-1.5-pro + litellm_params: + model: gemini-1.5-pro + vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json" + vertex_project: "my-special-project" + vertex_location: "my-special-location: +``` + + + + + + + +### **Environment Variables** + +You can set: +- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly). +- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.) +- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials + +1. GOOGLE_APPLICATION_CREDENTIALS + +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" +``` + +2. VERTEXAI_LOCATION + +```bash +export VERTEXAI_LOCATION="us-central1" # can be any vertex location +``` + +3. VERTEXAI_PROJECT + +```bash +export VERTEXAI_PROJECT="my-test-project" # ONLY use if model project is different from service account project +``` + + ## Specifying Safety Settings In certain use-cases you may need to make calls to the models and pass [safety settigns](https://ai.google.dev/docs/safety_setting_gemini) different from the defaults. To do so, simple pass the `safety_settings` argument to `completion` or `acompletion`. For example: @@ -2303,97 +2393,6 @@ print("response from proxy", response) - - -## Authentication - vertex_project, vertex_location, etc. - -Set your vertex credentials via: -- dynamic params -OR -- env vars - - -### **Dynamic Params** - -You can set: -- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json -- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.) -- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials - -as dynamic params for a `litellm.completion` call. - - - - -```python -from litellm import completion -import json - -## GET CREDENTIALS -file_path = 'path/to/vertex_ai_service_account.json' - -# Load the JSON file -with open(file_path, 'r') as file: - vertex_credentials = json.load(file) - -# Convert to JSON string -vertex_credentials_json = json.dumps(vertex_credentials) - - -response = completion( - model="vertex_ai/gemini-pro", - messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}], - vertex_credentials=vertex_credentials_json, - vertex_project="my-special-project", - vertex_location="my-special-location" -) -``` - - - - -```yaml -model_list: - - model_name: gemini-1.5-pro - litellm_params: - model: gemini-1.5-pro - vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json" - vertex_project: "my-special-project" - vertex_location: "my-special-location: -``` - - - - - - - -### **Environment Variables** - -You can set: -- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly). -- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.) -- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials - -1. GOOGLE_APPLICATION_CREDENTIALS - -```bash -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" -``` - -2. VERTEXAI_LOCATION - -```bash -export VERTEXAI_LOCATION="us-central1" # can be any vertex location -``` - -3. VERTEXAI_PROJECT - -```bash -export VERTEXAI_PROJECT="my-test-project" # ONLY use if model project is different from service account project -``` - - ## Extra ### Using `GOOGLE_APPLICATION_CREDENTIALS` From f2e6c9c9d866a00b94ecb9a297dd8c5406664384 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:30:01 +0530 Subject: [PATCH 011/113] test: handle overloaded anthropic model error --- tests/llm_translation/test_optional_params.py | 9 --------- tests/local_testing/test_completion.py | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 33e708ff4..7fe8baeb5 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -952,12 +952,3 @@ def test_lm_studio_embedding_params(): drop_params=True, ) assert len(optional_params) == 0 - - -def test_vertex_ft_models_optional_params(): - optional_params = get_optional_params( - model="meta-llama/Llama-3.1-8B-Instruct", - custom_llm_provider="vertex_ai", - max_retries=3, - ) - assert "max_retries" not in optional_params diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 3ce4cb7d7..936d2edc9 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1268,6 +1268,8 @@ async def test_acompletion_claude2_1(): print(response.usage.completion_tokens) print(response["usage"]["completion_tokens"]) # print("new cost tracking") + except litellm.InternalServerError: + pytest.skip("model is overloaded.") except Exception as e: pytest.fail(f"Error occurred: {e}") From 59a9b71d216d942a28f3ca1ca1dec7e60652b711 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:50:08 +0530 Subject: [PATCH 012/113] build: fix test --- litellm/model_prices_and_context_window_backup.json | 6 ++---- tests/test_organizations.py | 9 ++++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f8dd86cbc..815672ff2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4674,8 +4674,7 @@ "litellm_provider": "bedrock", "mode": "chat", "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_vision": true + "supports_function_calling": true }, "us.anthropic.claude-3-opus-20240229-v1:0": { "max_tokens": 4096, @@ -4741,8 +4740,7 @@ "output_cost_per_token": 0.000005, "litellm_provider": "bedrock", "mode": "chat", - "supports_function_calling": true, - "supports_vision": true + "supports_function_calling": true }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "max_tokens": 4096, diff --git a/tests/test_organizations.py b/tests/test_organizations.py index d62380b4a..947f26019 100644 --- a/tests/test_organizations.py +++ b/tests/test_organizations.py @@ -29,6 +29,7 @@ async def new_organization(session, i, organization_alias, max_budget=None): return await response.json() + async def list_organization(session, i): url = "http://0.0.0.0:4000/organization/list" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} @@ -46,6 +47,7 @@ async def list_organization(session, i): return await response.json() + @pytest.mark.asyncio async def test_organization_new(): """ @@ -61,6 +63,7 @@ async def test_organization_new(): ] await asyncio.gather(*tasks) + @pytest.mark.asyncio async def test_organization_list(): """ @@ -77,8 +80,8 @@ async def test_organization_list(): ] await asyncio.gather(*tasks) - response_json = await list_organization(session) + response_json = await list_organization(session, i=0) print(len(response_json)) - if len(response_json)==0: - raise Exception(f"Return empty list of organization") + if len(response_json) == 0: + raise Exception("Return empty list of organization") From 4ddbf1395f59263e6cbc950a2261f34361d818d5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 05:53:43 +0530 Subject: [PATCH 013/113] test: remove duplicate test --- tests/local_testing/test_completion.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 936d2edc9..12f7b65cc 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1222,32 +1222,6 @@ def test_completion_mistral_api_modified_input(): pytest.fail(f"Error occurred: {e}") -def test_completion_claude2_1(): - try: - litellm.set_verbose = True - print("claude2.1 test request") - messages = [ - { - "role": "system", - "content": "Your goal is generate a joke on the topic user gives.", - }, - {"role": "user", "content": "Generate a 3 liner joke for me"}, - ] - # test without max tokens - response = completion(model="claude-2.1", messages=messages) - # Add any assertions here to check the response - print(response) - print(response.usage) - print(response.usage.completion_tokens) - print(response["usage"]["completion_tokens"]) - # print("new cost tracking") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_claude2_1() - - @pytest.mark.asyncio async def test_acompletion_claude2_1(): try: From 3c6fe21935fe37bac0e2250da8400a440d8f62cb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 20:25:27 -0800 Subject: [PATCH 014/113] (Feat) Add provider specific budget routing (#6817) * add ProviderBudgetConfig * working test_provider_budgets_e2e_test * test_provider_budgets_e2e_test_expect_to_fail * use 1 cache read for getting provider spend * test_provider_budgets_e2e_test * add doc on provider budgets * clean up provider budgets * unit testing for provider budget routing * use as flag, not routing strat * fix init provider budget routing * use async_filter_deployments * fix test provider budgets * doc provider budget routing * doc provider budget routing * fix docs changes * fix comment --- .../docs/proxy/provider_budget_routing.md | 64 +++++ docs/my-website/sidebars.js | 2 +- litellm/router.py | 20 +- litellm/router_strategy/provider_budgets.py | 219 ++++++++++++++++++ litellm/types/router.py | 9 + tests/local_testing/test_provider_budgets.py | 209 +++++++++++++++++ 6 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 docs/my-website/docs/proxy/provider_budget_routing.md create mode 100644 litellm/router_strategy/provider_budgets.py create mode 100644 tests/local_testing/test_provider_budgets.py diff --git a/docs/my-website/docs/proxy/provider_budget_routing.md b/docs/my-website/docs/proxy/provider_budget_routing.md new file mode 100644 index 000000000..a945ef89a --- /dev/null +++ b/docs/my-website/docs/proxy/provider_budget_routing.md @@ -0,0 +1,64 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Provider Budget Routing +Use this to set budgets for LLM Providers - example $100/day for OpenAI, $100/day for Azure. + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/chatgpt-functioncalling + api_key: os.environ/AZURE_API_KEY + api_version: os.environ/AZURE_API_VERSION + api_base: os.environ/AZURE_API_BASE + +router_settings: + redis_host: + redis_password: + redis_port: + provider_budget_config: + openai: + budget_limit: 0.000000000001 # float of $ value budget for time period + time_period: 1d # can be 1d, 2d, 30d + azure: + budget_limit: 100 + time_period: 1d + anthropic: + budget_limit: 100 + time_period: 10d + vertexai: + budget_limit: 100 + time_period: 12d + gemini: + budget_limit: 100 + time_period: 12d + +general_settings: + master_key: sk-1234 +``` + + +#### How provider-budget-routing works + +1. **Budget Tracking**: + - Uses Redis to track spend for each provider + - Tracks spend over specified time periods (e.g., "1d", "30d") + - Automatically resets spend after time period expires + +2. **Routing Logic**: + - Routes requests to providers under their budget limits + - Skips providers that have exceeded their budget + - If all providers exceed budget, raises an error + +3. **Supported Time Periods**: + - Format: "Xd" where X is number of days + - Examples: "1d" (1 day), "30d" (30 days) + +4. **Requirements**: + - Redis required for tracking spend across instances + - Provider names must be litellm provider names. See [Supported Providers](https://docs.litellm.ai/docs/providers) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 107a877da..50cc83c08 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -100,7 +100,7 @@ const sidebars = { { type: "category", label: "Routing", - items: ["proxy/load_balancing", "proxy/tag_routing", "proxy/team_based_routing", "proxy/customer_routing",], + items: ["proxy/load_balancing", "proxy/tag_routing", "proxy/provider_budget_routing", "proxy/team_based_routing", "proxy/customer_routing",], }, { type: "category", diff --git a/litellm/router.py b/litellm/router.py index 97065bc85..f724c96c4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -59,6 +59,7 @@ from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 +from litellm.router_strategy.provider_budgets import ProviderBudgetLimiting from litellm.router_strategy.simple_shuffle import simple_shuffle from litellm.router_strategy.tag_based_routing import get_deployments_for_tag from litellm.router_utils.batch_utils import ( @@ -119,6 +120,7 @@ from litellm.types.router import ( LiteLLMParamsTypedDict, ModelGroupInfo, ModelInfo, + ProviderBudgetConfigType, RetryPolicy, RouterErrors, RouterGeneralSettings, @@ -235,7 +237,8 @@ class Router: "cost-based-routing", "usage-based-routing-v2", ] = "simple-shuffle", - routing_strategy_args: dict = {}, # just for latency-based routing + routing_strategy_args: dict = {}, # just for latency-based + provider_budget_config: Optional[ProviderBudgetConfigType] = None, semaphore: Optional[asyncio.Semaphore] = None, alerting_config: Optional[AlertingConfig] = None, router_general_settings: Optional[ @@ -272,6 +275,7 @@ class Router: routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle". routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}. alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None. + provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. Returns: Router: An instance of the litellm.Router class. @@ -517,6 +521,12 @@ class Router: ) self.service_logger_obj = ServiceLogging() self.routing_strategy_args = routing_strategy_args + self.provider_budget_config = provider_budget_config + if self.provider_budget_config is not None: + self.provider_budget_logger = ProviderBudgetLimiting( + router_cache=self.cache, + provider_budget_config=self.provider_budget_config, + ) self.retry_policy: Optional[RetryPolicy] = None if retry_policy is not None: if isinstance(retry_policy, dict): @@ -5109,6 +5119,14 @@ class Router: healthy_deployments=healthy_deployments, ) + if self.provider_budget_config is not None: + healthy_deployments = ( + await self.provider_budget_logger.async_filter_deployments( + healthy_deployments=healthy_deployments, + request_kwargs=request_kwargs, + ) + ) + if len(healthy_deployments) == 0: exception = await async_raise_no_deployment_exception( litellm_router_instance=self, diff --git a/litellm/router_strategy/provider_budgets.py b/litellm/router_strategy/provider_budgets.py new file mode 100644 index 000000000..c1805fea9 --- /dev/null +++ b/litellm/router_strategy/provider_budgets.py @@ -0,0 +1,219 @@ +""" +Provider budget limiting + +Use this if you want to set $ budget limits for each provider. + +Note: This is a filter, like tag-routing. Meaning it will accept healthy deployments and then filter out deployments that have exceeded their budget limit. + +This means you can use this with weighted-pick, lowest-latency, simple-shuffle, routing etc + +Example: +``` +openai: + budget_limit: 0.000000000001 + time_period: 1d +anthropic: + budget_limit: 100 + time_period: 7d +``` +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypedDict, Union + +import litellm +from litellm._logging import verbose_router_logger +from litellm.caching.caching import DualCache +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs +from litellm.types.router import ( + LiteLLM_Params, + ProviderBudgetConfigType, + ProviderBudgetInfo, +) +from litellm.types.utils import StandardLoggingPayload + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span = _Span +else: + Span = Any + + +class ProviderBudgetLimiting(CustomLogger): + def __init__(self, router_cache: DualCache, provider_budget_config: dict): + self.router_cache = router_cache + self.provider_budget_config: ProviderBudgetConfigType = provider_budget_config + verbose_router_logger.debug( + f"Initalized Provider budget config: {self.provider_budget_config}" + ) + + # Add self to litellm callbacks if it's a list + if isinstance(litellm.callbacks, list): + litellm.callbacks.append(self) # type: ignore + + async def async_filter_deployments( + self, + healthy_deployments: Union[List[Dict[str, Any]], Dict[str, Any]], + request_kwargs: Optional[Dict] = None, + ): + """ + Filter out deployments that have exceeded their provider budget limit. + + + Example: + if deployment = openai/gpt-3.5-turbo + and openai spend > openai budget limit + then skip this deployment + """ + + # If a single deployment is passed, convert it to a list + if isinstance(healthy_deployments, dict): + healthy_deployments = [healthy_deployments] + + potential_deployments: List[Dict] = [] + + # Extract the parent OpenTelemetry span for tracing + parent_otel_span: Optional[Span] = _get_parent_otel_span_from_kwargs( + request_kwargs + ) + + # Collect all providers and their budget configs + # {"openai": ProviderBudgetInfo, "anthropic": ProviderBudgetInfo, "azure": None} + _provider_configs: Dict[str, Optional[ProviderBudgetInfo]] = {} + for deployment in healthy_deployments: + provider = self._get_llm_provider_for_deployment(deployment) + if provider is None: + continue + budget_config = self._get_budget_config_for_provider(provider) + _provider_configs[provider] = budget_config + + # Filter out providers without budget config + provider_configs: Dict[str, ProviderBudgetInfo] = { + provider: config + for provider, config in _provider_configs.items() + if config is not None + } + + # Build cache keys for batch retrieval + cache_keys = [] + for provider, config in provider_configs.items(): + cache_keys.append(f"provider_spend:{provider}:{config.time_period}") + + # Fetch current spend for all providers using batch cache + _current_spends = await self.router_cache.async_batch_get_cache( + keys=cache_keys, + parent_otel_span=parent_otel_span, + ) + current_spends: List = _current_spends or [0.0] * len(provider_configs) + + # Map providers to their current spend values + provider_spend_map: Dict[str, float] = {} + for idx, provider in enumerate(provider_configs.keys()): + provider_spend_map[provider] = float(current_spends[idx] or 0.0) + + # Filter healthy deployments based on budget constraints + for deployment in healthy_deployments: + provider = self._get_llm_provider_for_deployment(deployment) + if provider is None: + continue + budget_config = provider_configs.get(provider) + + if not budget_config: + continue + + current_spend = provider_spend_map.get(provider, 0.0) + budget_limit = budget_config.budget_limit + + verbose_router_logger.debug( + f"Current spend for {provider}: {current_spend}, budget limit: {budget_limit}" + ) + + if current_spend >= budget_limit: + verbose_router_logger.debug( + f"Skipping deployment {deployment} for provider {provider} as spend limit exceeded" + ) + continue + + potential_deployments.append(deployment) + + return potential_deployments + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + """ + Increment provider spend in DualCache (InMemory + Redis) + + Handles saving current provider spend to Redis. + + Spend is stored as: + provider_spend:{provider}:{time_period} + ex. provider_spend:openai:1d + ex. provider_spend:anthropic:7d + + The time period is tracked for time_periods set in the provider budget config. + """ + verbose_router_logger.debug("in ProviderBudgetLimiting.async_log_success_event") + standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object", None + ) + if standard_logging_payload is None: + raise ValueError("standard_logging_payload is required") + + response_cost: float = standard_logging_payload.get("response_cost", 0) + + custom_llm_provider: str = kwargs.get("litellm_params", {}).get( + "custom_llm_provider", None + ) + if custom_llm_provider is None: + raise ValueError("custom_llm_provider is required") + + budget_config = self._get_budget_config_for_provider(custom_llm_provider) + if budget_config is None: + raise ValueError( + f"No budget config found for provider {custom_llm_provider}, self.provider_budget_config: {self.provider_budget_config}" + ) + + spend_key = f"provider_spend:{custom_llm_provider}:{budget_config.time_period}" + ttl_seconds = self.get_ttl_seconds(budget_config.time_period) + verbose_router_logger.debug( + f"Incrementing spend for {spend_key} by {response_cost}, ttl: {ttl_seconds}" + ) + # Increment the spend in Redis and set TTL + await self.router_cache.async_increment_cache( + key=spend_key, + value=response_cost, + ttl=ttl_seconds, + ) + verbose_router_logger.debug( + f"Incremented spend for {spend_key} by {response_cost}, ttl: {ttl_seconds}" + ) + + def _get_budget_config_for_provider( + self, provider: str + ) -> Optional[ProviderBudgetInfo]: + return self.provider_budget_config.get(provider, None) + + def _get_llm_provider_for_deployment(self, deployment: Dict) -> Optional[str]: + try: + _litellm_params: LiteLLM_Params = LiteLLM_Params( + **deployment.get("litellm_params", {"model": ""}) + ) + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=_litellm_params.model, + litellm_params=_litellm_params, + ) + except Exception: + verbose_router_logger.error( + f"Error getting LLM provider for deployment: {deployment}" + ) + return None + return custom_llm_provider + + def get_ttl_seconds(self, time_period: str) -> int: + """ + Convert time period (e.g., '1d', '30d') to seconds for Redis TTL + """ + if time_period.endswith("d"): + days = int(time_period[:-1]) + return days * 24 * 60 * 60 + raise ValueError(f"Unsupported time period format: {time_period}") diff --git a/litellm/types/router.py b/litellm/types/router.py index bb93aaa63..f4d2b39ed 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -628,3 +628,12 @@ class RoutingStrategy(enum.Enum): COST_BASED = "cost-based-routing" USAGE_BASED_ROUTING_V2 = "usage-based-routing-v2" USAGE_BASED_ROUTING = "usage-based-routing" + PROVIDER_BUDGET_LIMITING = "provider-budget-routing" + + +class ProviderBudgetInfo(BaseModel): + time_period: str # e.g., '1d', '30d' + budget_limit: float + + +ProviderBudgetConfigType = Dict[str, ProviderBudgetInfo] diff --git a/tests/local_testing/test_provider_budgets.py b/tests/local_testing/test_provider_budgets.py new file mode 100644 index 000000000..5e685cae6 --- /dev/null +++ b/tests/local_testing/test_provider_budgets.py @@ -0,0 +1,209 @@ +import sys, os, asyncio, time, random +from datetime import datetime +import traceback +from dotenv import load_dotenv + +load_dotenv() +import os, copy + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import pytest +from litellm import Router +from litellm.router_strategy.provider_budgets import ProviderBudgetLimiting +from litellm.types.router import ( + RoutingStrategy, + ProviderBudgetConfigType, + ProviderBudgetInfo, +) +from litellm.caching.caching import DualCache +import logging +from litellm._logging import verbose_router_logger + +verbose_router_logger.setLevel(logging.DEBUG) + + +@pytest.mark.asyncio +async def test_provider_budgets_e2e_test(): + """ + Expected behavior: + - First request forced to OpenAI + - Hit OpenAI budget limit + - Next 3 requests all go to Azure + + """ + provider_budget_config: ProviderBudgetConfigType = { + "openai": ProviderBudgetInfo(time_period="1d", budget_limit=0.000000000001), + "azure": ProviderBudgetInfo(time_period="1d", budget_limit=100), + } + + router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + }, + "model_info": {"id": "azure-model-id"}, + }, + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { + "model": "openai/gpt-4o-mini", + }, + "model_info": {"id": "openai-model-id"}, + }, + ], + provider_budget_config=provider_budget_config, + redis_host=os.getenv("REDIS_HOST"), + redis_port=int(os.getenv("REDIS_PORT")), + redis_password=os.getenv("REDIS_PASSWORD"), + ) + + response = await router.acompletion( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="openai/gpt-4o-mini", + ) + print(response) + + await asyncio.sleep(0.5) + + for _ in range(3): + response = await router.acompletion( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="gpt-3.5-turbo", + ) + print(response) + + print("response.hidden_params", response._hidden_params) + + await asyncio.sleep(0.5) + + assert response._hidden_params.get("custom_llm_provider") == "azure" + + +@pytest.mark.asyncio +async def test_provider_budgets_e2e_test_expect_to_fail(): + """ + Expected behavior: + - first request passes, all subsequent requests fail + + """ + provider_budget_config: ProviderBudgetConfigType = { + "anthropic": ProviderBudgetInfo(time_period="1d", budget_limit=0.000000000001), + } + + router = Router( + model_list=[ + { + "model_name": "anthropic/*", # openai model name + "litellm_params": { + "model": "anthropic/*", + }, + }, + ], + redis_host=os.getenv("REDIS_HOST"), + redis_port=int(os.getenv("REDIS_PORT")), + redis_password=os.getenv("REDIS_PASSWORD"), + provider_budget_config=provider_budget_config, + ) + + response = await router.acompletion( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="anthropic/claude-3-5-sonnet-20240620", + ) + print(response) + + await asyncio.sleep(0.5) + + for _ in range(3): + with pytest.raises(Exception) as exc_info: + response = await router.acompletion( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="anthropic/claude-3-5-sonnet-20240620", + ) + print(response) + print("response.hidden_params", response._hidden_params) + + await asyncio.sleep(0.5) + # Verify the error is related to budget exceeded + + +def test_get_ttl_seconds(): + """ + Test the get_ttl_seconds helper method" + + """ + provider_budget = ProviderBudgetLimiting( + router_cache=DualCache(), provider_budget_config={} + ) + + assert provider_budget.get_ttl_seconds("1d") == 86400 # 1 day in seconds + assert provider_budget.get_ttl_seconds("7d") == 604800 # 7 days in seconds + assert provider_budget.get_ttl_seconds("30d") == 2592000 # 30 days in seconds + + with pytest.raises(ValueError, match="Unsupported time period format"): + provider_budget.get_ttl_seconds("1h") + + +def test_get_llm_provider_for_deployment(): + """ + Test the _get_llm_provider_for_deployment helper method + + """ + provider_budget = ProviderBudgetLimiting( + router_cache=DualCache(), provider_budget_config={} + ) + + # Test OpenAI deployment + openai_deployment = {"litellm_params": {"model": "openai/gpt-4"}} + assert ( + provider_budget._get_llm_provider_for_deployment(openai_deployment) == "openai" + ) + + # Test Azure deployment + azure_deployment = { + "litellm_params": { + "model": "azure/gpt-4", + "api_key": "test", + "api_base": "test", + } + } + assert provider_budget._get_llm_provider_for_deployment(azure_deployment) == "azure" + + # should not raise error for unknown deployment + unknown_deployment = {} + assert provider_budget._get_llm_provider_for_deployment(unknown_deployment) is None + + +def test_get_budget_config_for_provider(): + """ + Test the _get_budget_config_for_provider helper method + + """ + config = { + "openai": ProviderBudgetInfo(time_period="1d", budget_limit=100), + "anthropic": ProviderBudgetInfo(time_period="7d", budget_limit=500), + } + + provider_budget = ProviderBudgetLimiting( + router_cache=DualCache(), provider_budget_config=config + ) + + # Test existing providers + openai_config = provider_budget._get_budget_config_for_provider("openai") + assert openai_config is not None + assert openai_config.time_period == "1d" + assert openai_config.budget_limit == 100 + + anthropic_config = provider_budget._get_budget_config_for_provider("anthropic") + assert anthropic_config is not None + assert anthropic_config.time_period == "7d" + assert anthropic_config.budget_limit == 500 + + # Test non-existent provider + assert provider_budget._get_budget_config_for_provider("unknown") is None From 7463dab9c66e706f17c2fa7ce195c30b515b9581 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 21:25:08 -0800 Subject: [PATCH 015/113] (feat) provider budget routing improvements (#6827) * minor fix for provider budget * fix raise good error message when budget crossed for provider budget * fix test provider budgets * test provider budgets * feat - emit llm provider spend on prometheus * test_prometheus_metric_tracking * doc provider budgets --- .../docs/proxy/provider_budget_routing.md | 100 ++++++++++++++++-- litellm/integrations/prometheus.py | 20 ++++ litellm/proxy/proxy_config.yaml | 20 ++-- litellm/router_strategy/provider_budgets.py | 57 +++++++++- litellm/router_utils/cooldown_callbacks.py | 3 + litellm/types/router.py | 3 + tests/local_testing/test_provider_budgets.py | 78 +++++++++++++- 7 files changed, 261 insertions(+), 20 deletions(-) diff --git a/docs/my-website/docs/proxy/provider_budget_routing.md b/docs/my-website/docs/proxy/provider_budget_routing.md index a945ef89a..fea3f483c 100644 --- a/docs/my-website/docs/proxy/provider_budget_routing.md +++ b/docs/my-website/docs/proxy/provider_budget_routing.md @@ -4,18 +4,16 @@ import TabItem from '@theme/TabItem'; # Provider Budget Routing Use this to set budgets for LLM Providers - example $100/day for OpenAI, $100/day for Azure. +## Quick Start + +Set provider budgets in your `proxy_config.yaml` file +### Proxy Config setup ```yaml model_list: - model_name: gpt-3.5-turbo litellm_params: model: openai/gpt-3.5-turbo api_key: os.environ/OPENAI_API_KEY - - model_name: gpt-3.5-turbo - litellm_params: - model: azure/chatgpt-functioncalling - api_key: os.environ/AZURE_API_KEY - api_version: os.environ/AZURE_API_VERSION - api_base: os.environ/AZURE_API_BASE router_settings: redis_host: @@ -42,8 +40,66 @@ general_settings: master_key: sk-1234 ``` +### Make a test request -#### How provider-budget-routing works +We expect the first request to succeed, and the second request to fail since we cross the budget for `openai` + + +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + + + + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "hi my name is test request"} + ] + }' +``` + + + + +Expect this to fail since since `ishaan@berri.ai` in the request is PII + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "hi my name is test request"} + ] + }' +``` + +Expected response on failure + +```json +{ + "error": { + "message": "No deployments available - crossed budget for provider: Exceeded budget for provider openai: 0.0007350000000000001 >= 1e-12", + "type": "None", + "param": "None", + "code": "429" + } +} +``` + + + + + + + + +## How provider budget routing works 1. **Budget Tracking**: - Uses Redis to track spend for each provider @@ -62,3 +118,33 @@ general_settings: 4. **Requirements**: - Redis required for tracking spend across instances - Provider names must be litellm provider names. See [Supported Providers](https://docs.litellm.ai/docs/providers) + +## Monitoring Provider Remaining Budget + +LiteLLM will emit the following metric on Prometheus to track the remaining budget for each provider + +This metric indicates the remaining budget for a provider in dollars (USD) + +``` +litellm_provider_remaining_budget_metric{api_provider="openai"} 10 +``` + + +## Spec for provider_budget_config + +The `provider_budget_config` is a dictionary where: +- **Key**: Provider name (string) - Must be a valid [LiteLLM provider name](https://docs.litellm.ai/docs/providers) +- **Value**: Budget configuration object with the following parameters: + - `budget_limit`: Float value representing the budget in USD + - `time_period`: String in the format "Xd" where X is the number of days (e.g., "1d", "30d") + +Example structure: +```yaml +provider_budget_config: + openai: + budget_limit: 100.0 # $100 USD + time_period: "1d" # 1 day period + azure: + budget_limit: 500.0 # $500 USD + time_period: "30d" # 30 day period +``` \ No newline at end of file diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index cbeb4d336..bb28719a3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -228,6 +228,13 @@ class PrometheusLogger(CustomLogger): "api_key_alias", ], ) + # llm api provider budget metrics + self.litellm_provider_remaining_budget_metric = Gauge( + "litellm_provider_remaining_budget_metric", + "Remaining budget for provider - used when you set provider budget limits", + labelnames=["api_provider"], + ) + # Get all keys _logged_llm_labels = [ "litellm_model_name", @@ -1130,6 +1137,19 @@ class PrometheusLogger(CustomLogger): litellm_model_name, model_id, api_base, api_provider, exception_status ).inc() + def track_provider_remaining_budget( + self, provider: str, spend: float, budget_limit: float + ): + """ + Track provider remaining budget in Prometheus + """ + self.litellm_provider_remaining_budget_metric.labels(provider).set( + self._safe_get_remaining_budget( + max_budget=budget_limit, + spend=spend, + ) + ) + def _safe_get_remaining_budget( self, max_budget: Optional[float], spend: Optional[float] ) -> float: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 71e3dee0e..3fc7ecfe2 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,14 +1,18 @@ model_list: - - model_name: fake-openai-endpoint + - model_name: gpt-4o litellm_params: - model: openai/fake + model: openai/gpt-4o api_key: os.environ/OPENAI_API_KEY - api_base: https://exampleopenaiendpoint-production.up.railway.app/ +router_settings: + provider_budget_config: + openai: + budget_limit: 0.000000000001 # float of $ value budget for time period + time_period: 1d # can be 1d, 2d, 30d + azure: + budget_limit: 100 + time_period: 1d -general_settings: - key_management_system: "aws_secret_manager" - key_management_settings: - store_virtual_keys: true - access_mode: "write_only" +litellm_settings: + callbacks: ["prometheus"] diff --git a/litellm/router_strategy/provider_budgets.py b/litellm/router_strategy/provider_budgets.py index c1805fea9..23d8b6c39 100644 --- a/litellm/router_strategy/provider_budgets.py +++ b/litellm/router_strategy/provider_budgets.py @@ -25,10 +25,14 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs +from litellm.router_utils.cooldown_callbacks import ( + _get_prometheus_logger_from_callbacks, +) from litellm.types.router import ( LiteLLM_Params, ProviderBudgetConfigType, ProviderBudgetInfo, + RouterErrors, ) from litellm.types.utils import StandardLoggingPayload @@ -43,6 +47,20 @@ else: class ProviderBudgetLimiting(CustomLogger): def __init__(self, router_cache: DualCache, provider_budget_config: dict): self.router_cache = router_cache + + # cast elements of provider_budget_config to ProviderBudgetInfo + for provider, config in provider_budget_config.items(): + if config is None: + raise ValueError( + f"No budget config found for provider {provider}, provider_budget_config: {provider_budget_config}" + ) + + if not isinstance(config, ProviderBudgetInfo): + provider_budget_config[provider] = ProviderBudgetInfo( + budget_limit=config.get("budget_limit"), + time_period=config.get("time_period"), + ) + self.provider_budget_config: ProviderBudgetConfigType = provider_budget_config verbose_router_logger.debug( f"Initalized Provider budget config: {self.provider_budget_config}" @@ -71,6 +89,10 @@ class ProviderBudgetLimiting(CustomLogger): if isinstance(healthy_deployments, dict): healthy_deployments = [healthy_deployments] + # Don't do any filtering if there are no healthy deployments + if len(healthy_deployments) == 0: + return healthy_deployments + potential_deployments: List[Dict] = [] # Extract the parent OpenTelemetry span for tracing @@ -113,6 +135,7 @@ class ProviderBudgetLimiting(CustomLogger): provider_spend_map[provider] = float(current_spends[idx] or 0.0) # Filter healthy deployments based on budget constraints + deployment_above_budget_info: str = "" # used to return in error message for deployment in healthy_deployments: provider = self._get_llm_provider_for_deployment(deployment) if provider is None: @@ -128,15 +151,25 @@ class ProviderBudgetLimiting(CustomLogger): verbose_router_logger.debug( f"Current spend for {provider}: {current_spend}, budget limit: {budget_limit}" ) + self._track_provider_remaining_budget_prometheus( + provider=provider, + spend=current_spend, + budget_limit=budget_limit, + ) if current_spend >= budget_limit: - verbose_router_logger.debug( - f"Skipping deployment {deployment} for provider {provider} as spend limit exceeded" - ) + debug_msg = f"Exceeded budget for provider {provider}: {current_spend} >= {budget_limit}" + verbose_router_logger.debug(debug_msg) + deployment_above_budget_info += f"{debug_msg}\n" continue potential_deployments.append(deployment) + if len(potential_deployments) == 0: + raise ValueError( + f"{RouterErrors.no_deployments_with_provider_budget_routing.value}: {deployment_above_budget_info}" + ) + return potential_deployments async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -217,3 +250,21 @@ class ProviderBudgetLimiting(CustomLogger): days = int(time_period[:-1]) return days * 24 * 60 * 60 raise ValueError(f"Unsupported time period format: {time_period}") + + def _track_provider_remaining_budget_prometheus( + self, provider: str, spend: float, budget_limit: float + ): + """ + Optional helper - emit provider remaining budget metric to Prometheus + + This is helpful for debugging and monitoring provider budget limits. + """ + from litellm.integrations.prometheus import PrometheusLogger + + prometheus_logger = _get_prometheus_logger_from_callbacks() + if prometheus_logger: + prometheus_logger.track_provider_remaining_budget( + provider=provider, + spend=spend, + budget_limit=budget_limit, + ) diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py index 7df2b2d6b..f6465d135 100644 --- a/litellm/router_utils/cooldown_callbacks.py +++ b/litellm/router_utils/cooldown_callbacks.py @@ -88,6 +88,9 @@ def _get_prometheus_logger_from_callbacks() -> Optional[PrometheusLogger]: """ from litellm.integrations.prometheus import PrometheusLogger + for _callback in litellm._async_success_callback: + if isinstance(_callback, PrometheusLogger): + return _callback for _callback in litellm.callbacks: if isinstance(_callback, PrometheusLogger): return _callback diff --git a/litellm/types/router.py b/litellm/types/router.py index f4d2b39ed..f91155a22 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -434,6 +434,9 @@ class RouterErrors(enum.Enum): no_deployments_with_tag_routing = ( "Not allowed to access model due to tags configuration" ) + no_deployments_with_provider_budget_routing = ( + "No deployments available - crossed budget for provider" + ) class AllowedFailsPolicy(BaseModel): diff --git a/tests/local_testing/test_provider_budgets.py b/tests/local_testing/test_provider_budgets.py index 5e685cae6..0c1995d43 100644 --- a/tests/local_testing/test_provider_budgets.py +++ b/tests/local_testing/test_provider_budgets.py @@ -20,6 +20,7 @@ from litellm.types.router import ( from litellm.caching.caching import DualCache import logging from litellm._logging import verbose_router_logger +import litellm verbose_router_logger.setLevel(logging.DEBUG) @@ -93,8 +94,14 @@ async def test_provider_budgets_e2e_test_expect_to_fail(): - first request passes, all subsequent requests fail """ - provider_budget_config: ProviderBudgetConfigType = { - "anthropic": ProviderBudgetInfo(time_period="1d", budget_limit=0.000000000001), + + # Note: We intentionally use a dictionary with string keys for budget_limit and time_period + # we want to test that the router can handle type conversion, since the proxy config yaml passes these values as a dictionary + provider_budget_config = { + "anthropic": { + "budget_limit": 0.000000000001, + "time_period": "1d", + } } router = Router( @@ -132,6 +139,8 @@ async def test_provider_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(0.5) # Verify the error is related to budget exceeded + assert "Exceeded budget for provider" in str(exc_info.value) + def test_get_ttl_seconds(): """ @@ -207,3 +216,68 @@ def test_get_budget_config_for_provider(): # Test non-existent provider assert provider_budget._get_budget_config_for_provider("unknown") is None + + +@pytest.mark.asyncio +async def test_prometheus_metric_tracking(): + """ + Test that the Prometheus metric for provider budget is tracked correctly + """ + from unittest.mock import MagicMock + from litellm.integrations.prometheus import PrometheusLogger + + # Create a mock PrometheusLogger + mock_prometheus = MagicMock(spec=PrometheusLogger) + + # Setup provider budget limiting + provider_budget = ProviderBudgetLimiting( + router_cache=DualCache(), + provider_budget_config={ + "openai": ProviderBudgetInfo(time_period="1d", budget_limit=100) + }, + ) + + litellm._async_success_callback = [mock_prometheus] + + provider_budget_config: ProviderBudgetConfigType = { + "openai": ProviderBudgetInfo(time_period="1d", budget_limit=0.000000000001), + "azure": ProviderBudgetInfo(time_period="1d", budget_limit=100), + } + + router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { # params for litellm completion/embedding call + "model": "azure/chatgpt-v-2", + "api_key": os.getenv("AZURE_API_KEY"), + "api_version": os.getenv("AZURE_API_VERSION"), + "api_base": os.getenv("AZURE_API_BASE"), + }, + "model_info": {"id": "azure-model-id"}, + }, + { + "model_name": "gpt-3.5-turbo", # openai model name + "litellm_params": { + "model": "openai/gpt-4o-mini", + }, + "model_info": {"id": "openai-model-id"}, + }, + ], + provider_budget_config=provider_budget_config, + redis_host=os.getenv("REDIS_HOST"), + redis_port=int(os.getenv("REDIS_PORT")), + redis_password=os.getenv("REDIS_PASSWORD"), + ) + + response = await router.acompletion( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="openai/gpt-4o-mini", + mock_response="hi", + ) + print(response) + + await asyncio.sleep(0.5) + + # Verify the mock was called correctly + mock_prometheus.track_provider_remaining_budget.assert_called_once() From 8b92e4f77a2d87df34998f837a2f87b24968d95d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 22:11:30 -0800 Subject: [PATCH 016/113] fix test_prometheus_metric_tracking --- tests/local_testing/test_provider_budgets.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/local_testing/test_provider_budgets.py b/tests/local_testing/test_provider_budgets.py index 0c1995d43..78c95246d 100644 --- a/tests/local_testing/test_provider_budgets.py +++ b/tests/local_testing/test_provider_budgets.py @@ -270,12 +270,15 @@ async def test_prometheus_metric_tracking(): redis_password=os.getenv("REDIS_PASSWORD"), ) - response = await router.acompletion( - messages=[{"role": "user", "content": "Hello, how are you?"}], - model="openai/gpt-4o-mini", - mock_response="hi", - ) - print(response) + try: + response = await router.acompletion( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="openai/gpt-4o-mini", + mock_response="hi", + ) + print(response) + except Exception as e: + print("error", e) await asyncio.sleep(0.5) From 8631f3bb606d6a6066f02974fbf498becd5f3eb4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 22:11:52 -0800 Subject: [PATCH 017/113] use correct name for test file --- .../{test_provider_budgets.py => test_router_provider_budgets.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/local_testing/{test_provider_budgets.py => test_router_provider_budgets.py} (100%) diff --git a/tests/local_testing/test_provider_budgets.py b/tests/local_testing/test_router_provider_budgets.py similarity index 100% rename from tests/local_testing/test_provider_budgets.py rename to tests/local_testing/test_router_provider_budgets.py From 132569dafcec7c71f915e62882231536d536b2ac Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 19 Nov 2024 22:38:45 -0800 Subject: [PATCH 018/113] ci/cd run again --- tests/local_testing/test_router_provider_budgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local_testing/test_router_provider_budgets.py b/tests/local_testing/test_router_provider_budgets.py index 78c95246d..46b9ee29e 100644 --- a/tests/local_testing/test_router_provider_budgets.py +++ b/tests/local_testing/test_router_provider_budgets.py @@ -8,7 +8,7 @@ import os, copy sys.path.insert( 0, os.path.abspath("../..") -) # Adds the parent directory to the system path +) # Adds the parent directory to the system-path import pytest from litellm import Router from litellm.router_strategy.provider_budgets import ProviderBudgetLimiting From f6ebba75381a8d9c27d37a58ba9347ae7012b0c3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 14:04:30 +0530 Subject: [PATCH 019/113] test: fix test --- tests/local_testing/test_whisper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/local_testing/test_whisper.py b/tests/local_testing/test_whisper.py index e2663a3ab..1d7b74087 100644 --- a/tests/local_testing/test_whisper.py +++ b/tests/local_testing/test_whisper.py @@ -51,9 +51,11 @@ from litellm import Router ), ], ) -@pytest.mark.parametrize("response_format", ["json", "vtt"]) +@pytest.mark.parametrize( + "response_format, timestamp_granularities", + [("json", None), ("vtt", None), ("verbose_json", ["word"])], +) @pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.parametrize("timestamp_granularities", [None, ["word"], ["segment"]]) @pytest.mark.asyncio async def test_transcription( model, api_key, api_base, response_format, sync_mode, timestamp_granularities From 6a816bceee035c0b5f81c3f2b37d49cd3b8b31c0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 14:13:07 +0530 Subject: [PATCH 020/113] test: fix test --- tests/test_organizations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_organizations.py b/tests/test_organizations.py index 947f26019..9bf6660d6 100644 --- a/tests/test_organizations.py +++ b/tests/test_organizations.py @@ -34,7 +34,7 @@ async def list_organization(session, i): url = "http://0.0.0.0:4000/organization/list" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} - async with session.post(url, headers=headers) as response: + async with session.get(url, headers=headers) as response: status = response.status response_json = await response.json() From 7d0e1f05acb4e814ed507e91ae31186e325a8bac Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 19:48:57 +0530 Subject: [PATCH 021/113] build: run new build --- litellm/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 3b4a99413..01804a071 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -554,7 +554,6 @@ def mock_completion( Raises: Exception: If an error occurs during the generation of the mock completion response. - Note: - This function is intended for testing or debugging purposes to generate mock completion responses. - If 'stream' is True, it returns a response that mimics the behavior of a streaming completion. From 361d010068da20d76d9074ead5e3fb912732c73a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 20 Nov 2024 22:02:33 +0530 Subject: [PATCH 022/113] test: update test to handle model overloaded error --- tests/local_testing/test_completion.py | 29 ++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 12f7b65cc..cf18e3673 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -4490,19 +4490,22 @@ async def test_dynamic_azure_params(stream, sync_mode): @pytest.mark.flaky(retries=3, delay=1) async def test_completion_ai21_chat(): litellm.set_verbose = True - response = await litellm.acompletion( - model="jamba-1.5-large", - user="ishaan", - tool_choice="auto", - seed=123, - messages=[{"role": "user", "content": "what does the document say"}], - documents=[ - { - "content": "hello world", - "metadata": {"source": "google", "author": "ishaan"}, - } - ], - ) + try: + response = await litellm.acompletion( + model="jamba-1.5-large", + user="ishaan", + tool_choice="auto", + seed=123, + messages=[{"role": "user", "content": "what does the document say"}], + documents=[ + { + "content": "hello world", + "metadata": {"source": "google", "author": "ishaan"}, + } + ], + ) + except litellm.InternalServerError: + pytest.skip("Model is overloaded") @pytest.mark.parametrize( From b0be5bf3a1e688b446e74b5482da7ef669b303c6 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 21 Nov 2024 00:57:58 +0530 Subject: [PATCH 023/113] LiteLLM Minor Fixes & Improvements (11/19/2024) (#6820) * fix(anthropic/chat/transformation.py): add json schema as values: json_schema fixes passing pydantic obj to anthropic Fixes https://github.com/BerriAI/litellm/issues/6766 * (feat): Add timestamp_granularities parameter to transcription API (#6457) * Add timestamp_granularities parameter to transcription API * add param to the local test * fix(databricks/chat.py): handle max_retries optional param handling for openai-like calls Fixes issue with calling finetuned vertex ai models via databricks route * build(ui/): add team admins via proxy ui * fix: fix linting error * test: fix test * docs(vertex.md): refactor docs * test: handle overloaded anthropic model error * test: remove duplicate test * test: fix test * test: update test to handle model overloaded error --------- Co-authored-by: Show <35062952+BrunooShow@users.noreply.github.com> --- docs/my-website/docs/providers/vertex.md | 181 +++++++++--------- litellm/llms/anthropic/chat/transformation.py | 2 +- litellm/llms/databricks/chat.py | 3 + litellm/main.py | 2 + ...odel_prices_and_context_window_backup.json | 34 ++-- litellm/utils.py | 1 + model_prices_and_context_window.json | 34 ++-- tests/llm_translation/base_llm_unit_tests.py | 43 ++++- .../test_anthropic_completion.py | 2 +- tests/llm_translation/test_optional_params.py | 3 +- .../test_amazing_vertex_completion.py | 6 +- tests/local_testing/test_completion.py | 57 ++---- tests/local_testing/test_whisper.py | 10 +- .../src/components/admins.tsx | 7 - ui/litellm-dashboard/src/components/teams.tsx | 8 +- 15 files changed, 200 insertions(+), 193 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 605762422..a7b363be1 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -572,6 +572,96 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server + +## Authentication - vertex_project, vertex_location, etc. + +Set your vertex credentials via: +- dynamic params +OR +- env vars + + +### **Dynamic Params** + +You can set: +- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json +- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.) +- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials + +as dynamic params for a `litellm.completion` call. + + + + +```python +from litellm import completion +import json + +## GET CREDENTIALS +file_path = 'path/to/vertex_ai_service_account.json' + +# Load the JSON file +with open(file_path, 'r') as file: + vertex_credentials = json.load(file) + +# Convert to JSON string +vertex_credentials_json = json.dumps(vertex_credentials) + + +response = completion( + model="vertex_ai/gemini-pro", + messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}], + vertex_credentials=vertex_credentials_json, + vertex_project="my-special-project", + vertex_location="my-special-location" +) +``` + + + + +```yaml +model_list: + - model_name: gemini-1.5-pro + litellm_params: + model: gemini-1.5-pro + vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json" + vertex_project: "my-special-project" + vertex_location: "my-special-location: +``` + + + + + + + +### **Environment Variables** + +You can set: +- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly). +- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.) +- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials + +1. GOOGLE_APPLICATION_CREDENTIALS + +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" +``` + +2. VERTEXAI_LOCATION + +```bash +export VERTEXAI_LOCATION="us-central1" # can be any vertex location +``` + +3. VERTEXAI_PROJECT + +```bash +export VERTEXAI_PROJECT="my-test-project" # ONLY use if model project is different from service account project +``` + + ## Specifying Safety Settings In certain use-cases you may need to make calls to the models and pass [safety settigns](https://ai.google.dev/docs/safety_setting_gemini) different from the defaults. To do so, simple pass the `safety_settings` argument to `completion` or `acompletion`. For example: @@ -2303,97 +2393,6 @@ print("response from proxy", response) - - -## Authentication - vertex_project, vertex_location, etc. - -Set your vertex credentials via: -- dynamic params -OR -- env vars - - -### **Dynamic Params** - -You can set: -- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json -- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.) -- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials - -as dynamic params for a `litellm.completion` call. - - - - -```python -from litellm import completion -import json - -## GET CREDENTIALS -file_path = 'path/to/vertex_ai_service_account.json' - -# Load the JSON file -with open(file_path, 'r') as file: - vertex_credentials = json.load(file) - -# Convert to JSON string -vertex_credentials_json = json.dumps(vertex_credentials) - - -response = completion( - model="vertex_ai/gemini-pro", - messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}], - vertex_credentials=vertex_credentials_json, - vertex_project="my-special-project", - vertex_location="my-special-location" -) -``` - - - - -```yaml -model_list: - - model_name: gemini-1.5-pro - litellm_params: - model: gemini-1.5-pro - vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json" - vertex_project: "my-special-project" - vertex_location: "my-special-location: -``` - - - - - - - -### **Environment Variables** - -You can set: -- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly). -- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.) -- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials - -1. GOOGLE_APPLICATION_CREDENTIALS - -```bash -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" -``` - -2. VERTEXAI_LOCATION - -```bash -export VERTEXAI_LOCATION="us-central1" # can be any vertex location -``` - -3. VERTEXAI_PROJECT - -```bash -export VERTEXAI_PROJECT="my-test-project" # ONLY use if model project is different from service account project -``` - - ## Extra ### Using `GOOGLE_APPLICATION_CREDENTIALS` diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1419d7ef2..ec981096c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -374,7 +374,7 @@ class AnthropicConfig: _input_schema["additionalProperties"] = True _input_schema["properties"] = {} else: - _input_schema["properties"] = json_schema + _input_schema["properties"] = {"values": json_schema} _tool = AnthropicMessagesTool(name="json_tool_call", input_schema=_input_schema) return _tool diff --git a/litellm/llms/databricks/chat.py b/litellm/llms/databricks/chat.py index eb0cb341e..79e885646 100644 --- a/litellm/llms/databricks/chat.py +++ b/litellm/llms/databricks/chat.py @@ -470,6 +470,9 @@ class DatabricksChatCompletion(BaseLLM): optional_params[k] = v stream: bool = optional_params.get("stream", None) or False + optional_params.pop( + "max_retries", None + ) # [TODO] add max retry support at llm api call level optional_params["stream"] = stream data = { diff --git a/litellm/main.py b/litellm/main.py index 01804a071..32055eb9d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4728,6 +4728,7 @@ def transcription( response_format: Optional[ Literal["json", "text", "srt", "verbose_json", "vtt"] ] = None, + timestamp_granularities: Optional[List[Literal["word", "segment"]]] = None, temperature: Optional[int] = None, # openai defaults this to 0 ## LITELLM PARAMS ## user: Optional[str] = None, @@ -4777,6 +4778,7 @@ def transcription( language=language, prompt=prompt, response_format=response_format, + timestamp_granularities=timestamp_granularities, temperature=temperature, custom_llm_provider=custom_llm_provider, drop_params=drop_params, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 815672ff2..1206f2642 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1884,7 +1884,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-haiku-20241022": { "max_tokens": 8192, @@ -1900,7 +1901,8 @@ "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, "supports_prompt_caching": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "supports_response_schema": true }, "claude-3-opus-20240229": { "max_tokens": 4096, @@ -1916,7 +1918,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 395, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-sonnet-20240229": { "max_tokens": 4096, @@ -1930,7 +1933,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20240620": { "max_tokens": 8192, @@ -1946,7 +1950,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20241022": { "max_tokens": 8192, @@ -1962,7 +1967,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "text-bison": { "max_tokens": 2048, @@ -3864,22 +3870,6 @@ "supports_function_calling": true, "tool_use_system_prompt_tokens": 264 }, - "anthropic/claude-3-5-sonnet-20241022": { - "max_tokens": 8192, - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_creation_input_token_cost": 0.00000375, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "anthropic", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, - "supports_prompt_caching": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "max_tokens": 8192, "max_input_tokens": 200000, diff --git a/litellm/utils.py b/litellm/utils.py index cb8a53354..2dce9db89 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2125,6 +2125,7 @@ def get_optional_params_transcription( prompt: Optional[str] = None, response_format: Optional[str] = None, temperature: Optional[int] = None, + timestamp_granularities: Optional[List[Literal["word", "segment"]]] = None, custom_llm_provider: Optional[str] = None, drop_params: Optional[bool] = None, **kwargs, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 815672ff2..1206f2642 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1884,7 +1884,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-haiku-20241022": { "max_tokens": 8192, @@ -1900,7 +1901,8 @@ "tool_use_system_prompt_tokens": 264, "supports_assistant_prefill": true, "supports_prompt_caching": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "supports_response_schema": true }, "claude-3-opus-20240229": { "max_tokens": 4096, @@ -1916,7 +1918,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 395, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-sonnet-20240229": { "max_tokens": 4096, @@ -1930,7 +1933,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20240620": { "max_tokens": 8192, @@ -1946,7 +1950,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "claude-3-5-sonnet-20241022": { "max_tokens": 8192, @@ -1962,7 +1967,8 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159, "supports_assistant_prefill": true, - "supports_prompt_caching": true + "supports_prompt_caching": true, + "supports_response_schema": true }, "text-bison": { "max_tokens": 2048, @@ -3864,22 +3870,6 @@ "supports_function_calling": true, "tool_use_system_prompt_tokens": 264 }, - "anthropic/claude-3-5-sonnet-20241022": { - "max_tokens": 8192, - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000015, - "cache_creation_input_token_cost": 0.00000375, - "cache_read_input_token_cost": 0.0000003, - "litellm_provider": "anthropic", - "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159, - "supports_assistant_prefill": true, - "supports_prompt_caching": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "max_tokens": 8192, "max_input_tokens": 200000, diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 955eed957..74fff60a4 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -42,11 +42,14 @@ class BaseLLMChatTest(ABC): "content": [{"type": "text", "text": "Hello, how are you?"}], } ] - response = litellm.completion( - **base_completion_call_args, - messages=messages, - ) - assert response is not None + try: + response = litellm.completion( + **base_completion_call_args, + messages=messages, + ) + assert response is not None + except litellm.InternalServerError: + pass # for OpenAI the content contains the JSON schema, so we need to assert that the content is not None assert response.choices[0].message.content is not None @@ -89,6 +92,36 @@ class BaseLLMChatTest(ABC): # relevant issue: https://github.com/BerriAI/litellm/issues/6741 assert response.choices[0].message.content is not None + def test_json_response_pydantic_obj(self): + from pydantic import BaseModel + from litellm.utils import supports_response_schema + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + class TestModel(BaseModel): + first_response: str + + base_completion_call_args = self.get_base_completion_call_args() + if not supports_response_schema(base_completion_call_args["model"], None): + pytest.skip("Model does not support response schema") + + try: + res = litellm.completion( + **base_completion_call_args, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + { + "role": "user", + "content": "What is the capital of France?", + }, + ], + response_format=TestModel, + ) + assert res is not None + except litellm.InternalServerError: + pytest.skip("Model is overloaded") + def test_json_response_format_stream(self): """ Test that the JSON response format with streaming is supported by the LLM API diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 8a788e0fb..c6181f1ba 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -657,7 +657,7 @@ def test_create_json_tool_call_for_response_format(): _input_schema = tool.get("input_schema") assert _input_schema is not None assert _input_schema.get("type") == "object" - assert _input_schema.get("properties") == custom_schema + assert _input_schema.get("properties") == {"values": custom_schema} assert "additionalProperties" not in _input_schema diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 029e91513..7fe8baeb5 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -923,7 +923,6 @@ def test_watsonx_text_top_k(): assert optional_params["top_k"] == 10 - def test_together_ai_model_params(): optional_params = get_optional_params( model="together_ai", custom_llm_provider="together_ai", logprobs=1 @@ -931,6 +930,7 @@ def test_together_ai_model_params(): print(optional_params) assert optional_params["logprobs"] == 1 + def test_forward_user_param(): from litellm.utils import get_supported_openai_params, get_optional_params @@ -943,6 +943,7 @@ def test_forward_user_param(): assert optional_params["metadata"]["user_id"] == "test_user" + def test_lm_studio_embedding_params(): optional_params = get_optional_params_embeddings( model="lm_studio/gemma2-9b-it", diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 3bf36dda8..f801a53ce 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -3129,9 +3129,12 @@ async def test_vertexai_embedding_finetuned(respx_mock: MockRouter): assert all(isinstance(x, float) for x in embedding["embedding"]) +@pytest.mark.parametrize("max_retries", [None, 3]) @pytest.mark.asyncio @pytest.mark.respx -async def test_vertexai_model_garden_model_completion(respx_mock: MockRouter): +async def test_vertexai_model_garden_model_completion( + respx_mock: MockRouter, max_retries +): """ Relevant issue: https://github.com/BerriAI/litellm/issues/6480 @@ -3189,6 +3192,7 @@ async def test_vertexai_model_garden_model_completion(respx_mock: MockRouter): messages=messages, vertex_project="633608382793", vertex_location="us-central1", + max_retries=max_retries, ) # Assert request was made correctly diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 3ce4cb7d7..cf18e3673 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1222,32 +1222,6 @@ def test_completion_mistral_api_modified_input(): pytest.fail(f"Error occurred: {e}") -def test_completion_claude2_1(): - try: - litellm.set_verbose = True - print("claude2.1 test request") - messages = [ - { - "role": "system", - "content": "Your goal is generate a joke on the topic user gives.", - }, - {"role": "user", "content": "Generate a 3 liner joke for me"}, - ] - # test without max tokens - response = completion(model="claude-2.1", messages=messages) - # Add any assertions here to check the response - print(response) - print(response.usage) - print(response.usage.completion_tokens) - print(response["usage"]["completion_tokens"]) - # print("new cost tracking") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_claude2_1() - - @pytest.mark.asyncio async def test_acompletion_claude2_1(): try: @@ -1268,6 +1242,8 @@ async def test_acompletion_claude2_1(): print(response.usage.completion_tokens) print(response["usage"]["completion_tokens"]) # print("new cost tracking") + except litellm.InternalServerError: + pytest.skip("model is overloaded.") except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -4514,19 +4490,22 @@ async def test_dynamic_azure_params(stream, sync_mode): @pytest.mark.flaky(retries=3, delay=1) async def test_completion_ai21_chat(): litellm.set_verbose = True - response = await litellm.acompletion( - model="jamba-1.5-large", - user="ishaan", - tool_choice="auto", - seed=123, - messages=[{"role": "user", "content": "what does the document say"}], - documents=[ - { - "content": "hello world", - "metadata": {"source": "google", "author": "ishaan"}, - } - ], - ) + try: + response = await litellm.acompletion( + model="jamba-1.5-large", + user="ishaan", + tool_choice="auto", + seed=123, + messages=[{"role": "user", "content": "what does the document say"}], + documents=[ + { + "content": "hello world", + "metadata": {"source": "google", "author": "ishaan"}, + } + ], + ) + except litellm.InternalServerError: + pytest.skip("Model is overloaded") @pytest.mark.parametrize( diff --git a/tests/local_testing/test_whisper.py b/tests/local_testing/test_whisper.py index f66ad8b13..1d7b74087 100644 --- a/tests/local_testing/test_whisper.py +++ b/tests/local_testing/test_whisper.py @@ -51,10 +51,15 @@ from litellm import Router ), ], ) -@pytest.mark.parametrize("response_format", ["json", "vtt"]) +@pytest.mark.parametrize( + "response_format, timestamp_granularities", + [("json", None), ("vtt", None), ("verbose_json", ["word"])], +) @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio -async def test_transcription(model, api_key, api_base, response_format, sync_mode): +async def test_transcription( + model, api_key, api_base, response_format, sync_mode, timestamp_granularities +): if sync_mode: transcript = litellm.transcription( model=model, @@ -62,6 +67,7 @@ async def test_transcription(model, api_key, api_base, response_format, sync_mod api_key=api_key, api_base=api_base, response_format=response_format, + timestamp_granularities=timestamp_granularities, drop_params=True, ) else: diff --git a/ui/litellm-dashboard/src/components/admins.tsx b/ui/litellm-dashboard/src/components/admins.tsx index 80c849ac1..f226d1c11 100644 --- a/ui/litellm-dashboard/src/components/admins.tsx +++ b/ui/litellm-dashboard/src/components/admins.tsx @@ -314,13 +314,6 @@ const AdminPanel: React.FC = ({ className="px-3 py-2 border rounded-md w-full" /> - {/*
OR
- - - */}
Add member diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index 90a29de32..11664bd02 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -381,7 +381,7 @@ const Team: React.FC = ({ if (accessToken != null && teams != null) { message.info("Adding Member"); const user_role: Member = { - role: "user", + role: formValues.role, user_email: formValues.user_email, user_id: formValues.user_id, }; @@ -809,6 +809,12 @@ const Team: React.FC = ({ className="px-3 py-2 border rounded-md w-full" /> + + + user + admin + +
Add member From a1f06de53df8a09cd36255dafe59974ac77cc50d Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 20 Nov 2024 17:18:29 -0500 Subject: [PATCH 024/113] Add gpt-4o-2024-11-20. (#6832) --- ...odel_prices_and_context_window_backup.json | 54 +++++++++++++++++++ model_prices_and_context_window.json | 54 +++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1206f2642..5e4f851e9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -197,6 +197,21 @@ "supports_vision": true, "supports_prompt_caching": true }, + "gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000010, + "cache_read_input_token_cost": 0.00000125, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, "gpt-4-turbo-preview": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -468,6 +483,19 @@ "supports_response_schema": true, "supports_vision": true }, + "ft:gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000375, + "output_cost_per_token": 0.000015, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "ft:gpt-4o-mini-2024-07-18": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -730,6 +758,19 @@ "supports_response_schema": true, "supports_vision": true }, + "azure/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "azure/gpt-4o-2024-05-13": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -756,6 +797,19 @@ "supports_response_schema": true, "supports_vision": true }, + "azure/global-standard/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000010, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "azure/global-standard/gpt-4o-mini": { "max_tokens": 16384, "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1206f2642..5e4f851e9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -197,6 +197,21 @@ "supports_vision": true, "supports_prompt_caching": true }, + "gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000010, + "cache_read_input_token_cost": 0.00000125, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_prompt_caching": true + }, "gpt-4-turbo-preview": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -468,6 +483,19 @@ "supports_response_schema": true, "supports_vision": true }, + "ft:gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000375, + "output_cost_per_token": 0.000015, + "litellm_provider": "openai", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "ft:gpt-4o-mini-2024-07-18": { "max_tokens": 16384, "max_input_tokens": 128000, @@ -730,6 +758,19 @@ "supports_response_schema": true, "supports_vision": true }, + "azure/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.00000275, + "output_cost_per_token": 0.000011, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "azure/gpt-4o-2024-05-13": { "max_tokens": 4096, "max_input_tokens": 128000, @@ -756,6 +797,19 @@ "supports_response_schema": true, "supports_vision": true }, + "azure/global-standard/gpt-4o-2024-11-20": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.000010, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_vision": true + }, "azure/global-standard/gpt-4o-mini": { "max_tokens": 16384, "max_input_tokens": 128000, From 689cd677c6cf23351413bfb3ee1df83d52fcf0ce Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 21 Nov 2024 04:06:06 +0530 Subject: [PATCH 025/113] Litellm dev 11 20 2024 (#6831) * feat(customer_endpoints.py): support passing budget duration via `/customer/new` endpoint Closes https://github.com/BerriAI/litellm/issues/5651 * docs: add missing params to swagger + api documentation test * docs: add documentation for all key endpoints documents all params on swagger * docs(internal_user_endpoints.py): document all /user/new params Ensures all params are documented * docs(team_endpoints.py): add missing documentation for team endpoints Ensures 100% param documentation on swagger * docs(organization_endpoints.py): document all org params Adds documentation for all params in org endpoint * docs(customer_endpoints.py): add coverage for all params on /customer endpoints ensures all /customer/* params are documented * ci(config.yml): add endpoint doc testing to ci/cd * fix: fix internal_user_endpoints.py * fix(internal_user_endpoints.py): support 'duration' param * fix(partner_models/main.py): fix anthropic re-raise exception on vertex * fix: fix pydantic obj --- .circleci/config.yml | 1 + .../vertex_ai_partner_models/main.py | 2 + litellm/proxy/_types.py | 138 ++++++++---- .../customer_endpoints.py | 67 +++++- .../internal_user_endpoints.py | 96 ++++---- .../key_management_endpoints.py | 9 + .../organization_endpoints.py | 79 ++++--- .../management_endpoints/team_endpoints.py | 17 ++ litellm/proxy/utils.py | 2 +- tests/documentation_tests/test_api_docs.py | 206 ++++++++++++++++++ .../test_key_generate_prisma.py | 2 +- 11 files changed, 480 insertions(+), 139 deletions(-) create mode 100644 tests/documentation_tests/test_api_docs.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d95a8c214..0a6327bb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -770,6 +770,7 @@ jobs: - run: python ./tests/code_coverage_tests/test_router_strategy_async.py - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - run: python ./tests/documentation_tests/test_env_keys.py + - run: python ./tests/documentation_tests/test_api_docs.py - run: helm lint ./deploy/charts/litellm-helm db_migration_disable_update_check: diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_partner_models/main.py index e8443e6f6..f335f53d9 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_partner_models/main.py @@ -236,4 +236,6 @@ class VertexAIPartnerModels(VertexBase): ) except Exception as e: + if hasattr(e, "status_code"): + raise e raise VertexAIError(status_code=500, message=str(e)) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f5851ded9..8b8dbf2e5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -623,6 +623,8 @@ class GenerateRequestBase(LiteLLMBase): Overlapping schema between key and user generate/update requests """ + key_alias: Optional[str] = None + duration: Optional[str] = None models: Optional[list] = [] spend: Optional[float] = 0 max_budget: Optional[float] = None @@ -635,13 +637,6 @@ class GenerateRequestBase(LiteLLMBase): budget_duration: Optional[str] = None allowed_cache_controls: Optional[list] = [] soft_budget: Optional[float] = None - - -class _GenerateKeyRequest(GenerateRequestBase): - key_alias: Optional[str] = None - key: Optional[str] = None - duration: Optional[str] = None - aliases: Optional[dict] = {} config: Optional[dict] = {} permissions: Optional[dict] = {} model_max_budget: Optional[dict] = ( @@ -654,6 +649,11 @@ class _GenerateKeyRequest(GenerateRequestBase): model_tpm_limit: Optional[dict] = None guardrails: Optional[List[str]] = None blocked: Optional[bool] = None + aliases: Optional[dict] = {} + + +class _GenerateKeyRequest(GenerateRequestBase): + key: Optional[str] = None class GenerateKeyRequest(_GenerateKeyRequest): @@ -719,7 +719,7 @@ class LiteLLM_ModelTable(LiteLLMBase): model_config = ConfigDict(protected_namespaces=()) -class NewUserRequest(_GenerateKeyRequest): +class NewUserRequest(GenerateRequestBase): max_budget: Optional[float] = None user_email: Optional[str] = None user_alias: Optional[str] = None @@ -786,7 +786,51 @@ class DeleteUserRequest(LiteLLMBase): AllowedModelRegion = Literal["eu", "us"] -class NewCustomerRequest(LiteLLMBase): +class BudgetNew(LiteLLMBase): + budget_id: Optional[str] = Field(default=None, description="The unique budget id.") + max_budget: Optional[float] = Field( + default=None, + description="Requests will fail if this budget (in USD) is exceeded.", + ) + soft_budget: Optional[float] = Field( + default=None, + description="Requests will NOT fail if this is exceeded. Will fire alerting though.", + ) + max_parallel_requests: Optional[int] = Field( + default=None, description="Max concurrent requests allowed for this budget id." + ) + tpm_limit: Optional[int] = Field( + default=None, description="Max tokens per minute, allowed for this budget id." + ) + rpm_limit: Optional[int] = Field( + default=None, description="Max requests per minute, allowed for this budget id." + ) + budget_duration: Optional[str] = Field( + default=None, + description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')", + ) + + +class BudgetRequest(LiteLLMBase): + budgets: List[str] + + +class BudgetDeleteRequest(LiteLLMBase): + id: str + + +class CustomerBase(LiteLLMBase): + user_id: str + alias: Optional[str] = None + spend: float = 0.0 + allowed_model_region: Optional[AllowedModelRegion] = None + default_model: Optional[str] = None + budget_id: Optional[str] = None + litellm_budget_table: Optional[BudgetNew] = None + blocked: bool = False + + +class NewCustomerRequest(BudgetNew): """ Create a new customer, allocate a budget to them """ @@ -794,7 +838,6 @@ class NewCustomerRequest(LiteLLMBase): user_id: str alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user - max_budget: Optional[float] = None budget_id: Optional[str] = None # give either a budget_id or max_budget allowed_model_region: Optional[AllowedModelRegion] = ( None # require all user requests to use models in this specific region @@ -1083,39 +1126,6 @@ class OrganizationRequest(LiteLLMBase): organizations: List[str] -class BudgetNew(LiteLLMBase): - budget_id: str = Field(default=None, description="The unique budget id.") - max_budget: Optional[float] = Field( - default=None, - description="Requests will fail if this budget (in USD) is exceeded.", - ) - soft_budget: Optional[float] = Field( - default=None, - description="Requests will NOT fail if this is exceeded. Will fire alerting though.", - ) - max_parallel_requests: Optional[int] = Field( - default=None, description="Max concurrent requests allowed for this budget id." - ) - tpm_limit: Optional[int] = Field( - default=None, description="Max tokens per minute, allowed for this budget id." - ) - rpm_limit: Optional[int] = Field( - default=None, description="Max requests per minute, allowed for this budget id." - ) - budget_duration: Optional[str] = Field( - default=None, - description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')", - ) - - -class BudgetRequest(LiteLLMBase): - budgets: List[str] - - -class BudgetDeleteRequest(LiteLLMBase): - id: str - - class KeyManagementSystem(enum.Enum): GOOGLE_KMS = "google_kms" AZURE_KEY_VAULT = "azure_key_vault" @@ -2081,3 +2091,45 @@ JWKKeyValue = Union[List[JWTKeyItem], JWTKeyItem] class JWKUrlResponse(TypedDict, total=False): keys: JWKKeyValue + + +class UserManagementEndpointParamDocStringEnums(str, enum.Enum): + user_id_doc_str = ( + "Optional[str] - Specify a user id. If not set, a unique id will be generated." + ) + user_alias_doc_str = ( + "Optional[str] - A descriptive name for you to know who this user id refers to." + ) + teams_doc_str = "Optional[list] - specify a list of team id's a user belongs to." + user_email_doc_str = "Optional[str] - Specify a user email." + send_invite_email_doc_str = ( + "Optional[bool] - Specify if an invite email should be sent." + ) + user_role_doc_str = """Optional[str] - Specify a user role - "proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer", "team", "customer". Info about each role here: `https://github.com/BerriAI/litellm/litellm/proxy/_types.py#L20`""" + max_budget_doc_str = """Optional[float] - Specify max budget for a given user.""" + budget_duration_doc_str = """Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo").""" + models_doc_str = """Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models)""" + tpm_limit_doc_str = ( + """Optional[int] - Specify tpm limit for a given user (Tokens per minute)""" + ) + rpm_limit_doc_str = ( + """Optional[int] - Specify rpm limit for a given user (Requests per minute)""" + ) + auto_create_key_doc_str = """bool - Default=True. Flag used for returning a key as part of the /user/new response""" + aliases_doc_str = """Optional[dict] - Model aliases for the user - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases)""" + config_doc_str = """Optional[dict] - [DEPRECATED PARAM] User-specific config.""" + allowed_cache_controls_doc_str = """Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request-""" + blocked_doc_str = ( + """Optional[bool] - [Not Implemented Yet] Whether the user is blocked.""" + ) + guardrails_doc_str = """Optional[List[str]] - [Not Implemented Yet] List of active guardrails for the user""" + permissions_doc_str = """Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.""" + metadata_doc_str = """Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }""" + max_parallel_requests_doc_str = """Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.""" + soft_budget_doc_str = """Optional[float] - Get alerts when user crosses given budget, doesn't block requests.""" + model_max_budget_doc_str = """Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)""" + model_rpm_limit_doc_str = """Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)""" + model_tpm_limit_doc_str = """Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)""" + spend_doc_str = """Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used.""" + team_id_doc_str = """Optional[str] - [DEPRECATED PARAM] The team id of the user. Default is None.""" + duration_doc_str = """Optional[str] - Duration for the key auto-created on `/user/new`. Default is None.""" diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index cb57619b9..48b01b0cb 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -1,3 +1,14 @@ +""" +CUSTOMER MANAGEMENT + +All /customer management endpoints + +/customer/new +/customer/info +/customer/update +/customer/delete +""" + #### END-USER/CUSTOMER MANAGEMENT #### import asyncio import copy @@ -129,6 +140,26 @@ async def unblock_user(data: BlockUsers): return {"blocked_users": litellm.blocked_user_list} +def new_budget_request(data: NewCustomerRequest) -> Optional[BudgetNew]: + """ + Return a new budget object if new budget params are passed. + """ + budget_params = BudgetNew.model_fields.keys() + budget_kv_pairs = {} + + # Get the actual values from the data object using getattr + for field_name in budget_params: + if field_name == "budget_id": + continue + value = getattr(data, field_name, None) + if value is not None: + budget_kv_pairs[field_name] = value + + if budget_kv_pairs: + return BudgetNew(**budget_kv_pairs) + return None + + @router.post( "/end_user/new", tags=["Customer Management"], @@ -157,6 +188,11 @@ async def new_end_user( - allowed_model_region: Optional[Union[Literal["eu"], Literal["us"]]] - Require all user requests to use models in this specific region. - default_model: Optional[str] - If no equivalent model in the allowed region, default all requests to this model. - metadata: Optional[dict] = Metadata for customer, store information for customer. Example metadata = {"data_training_opt_out": True} + - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). + - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) + - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) + - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer. + - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. - Allow specifying allowed regions @@ -223,14 +259,19 @@ async def new_end_user( new_end_user_obj: Dict = {} ## CREATE BUDGET ## if set - if data.max_budget is not None: - budget_record = await prisma_client.db.litellm_budgettable.create( - data={ - "max_budget": data.max_budget, - "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, # type: ignore - "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, - } - ) + _new_budget = new_budget_request(data) + if _new_budget is not None: + try: + budget_record = await prisma_client.db.litellm_budgettable.create( + data={ + **_new_budget.model_dump(exclude_unset=True), + "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, # type: ignore + "updated_by": user_api_key_dict.user_id + or litellm_proxy_admin_name, + } + ) + except Exception as e: + raise HTTPException(status_code=422, detail={"error": str(e)}) new_end_user_obj["budget_id"] = budget_record.budget_id elif data.budget_id is not None: @@ -239,16 +280,22 @@ async def new_end_user( _user_data = data.dict(exclude_none=True) for k, v in _user_data.items(): - if k != "max_budget" and k != "budget_id": + if k not in BudgetNew.model_fields.keys(): new_end_user_obj[k] = v ## WRITE TO DB ## end_user_record = await prisma_client.db.litellm_endusertable.create( - data=new_end_user_obj # type: ignore + data=new_end_user_obj, # type: ignore + include={"litellm_budget_table": True}, ) return end_user_record except Exception as e: + verbose_proxy_logger.exception( + "litellm.proxy.management_endpoints.customer_endpoints.new_end_user(): Exception occured - {}".format( + str(e) + ) + ) if "Unique constraint failed on the fields: (`user_id`)" in str(e): raise ProxyException( message=f"Customer already exists, passed user_id={data.user_id}. Please pass a new user_id.", diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 49ef25149..c69e255f2 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -102,11 +102,27 @@ async def new_user( - send_invite_email: Optional[bool] - Specify if an invite email should be sent. - user_role: Optional[str] - Specify a user role - "proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer", "team", "customer". Info about each role here: `https://github.com/BerriAI/litellm/litellm/proxy/_types.py#L20` - max_budget: Optional[float] - Specify max budget for a given user. - - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). + - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - models: Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models) - tpm_limit: Optional[int] - Specify tpm limit for a given user (Tokens per minute) - rpm_limit: Optional[int] - Specify rpm limit for a given user (Requests per minute) - auto_create_key: bool - Default=True. Flag used for returning a key as part of the /user/new response + - aliases: Optional[dict] - Model aliases for the user - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases) + - config: Optional[dict] - [DEPRECATED PARAM] User-specific config. + - allowed_cache_controls: Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request- + - blocked: Optional[bool] - [Not Implemented Yet] Whether the user is blocked. + - guardrails: Optional[List[str]] - [Not Implemented Yet] List of active guardrails for the user + - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. + - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } + - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. + - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. + - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) + - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) + - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) + - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). + - team_id: Optional[str] - [DEPRECATED PARAM] The team id of the user. Default is None. + - duration: Optional[str] - Duration for the key auto-created on `/user/new`. Default is None. + - key_alias: Optional[str] - Alias for the key auto-created on `/user/new`. Default is None. Returns: - key: (str) The generated api key for the user @@ -445,54 +461,36 @@ async def user_update( }' Parameters: - user_id: Optional[str] - Unique identifier for the user to update - - user_email: Optional[str] - Email address for the user - - password: Optional[str] - Password for the user - - user_role: Optional[Literal["proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer"]] - Role assigned to the user. Can be one of: - - proxy_admin: Full admin access - - proxy_admin_viewer: Read-only admin access - - internal_user: Standard internal user - - internal_user_viewer: Read-only internal user - - models: Optional[list] - List of model names the user is allowed to access - - spend: Optional[float] - Current spend amount for the user - - max_budget: Optional[float] - Maximum budget allowed for the user - - team_id: Optional[str] - ID of the team the user belongs to - - max_parallel_requests: Optional[int] - Maximum number of concurrent requests allowed - - metadata: Optional[dict] - Additional metadata associated with the user - - tpm_limit: Optional[int] - Maximum tokens per minute allowed - - rpm_limit: Optional[int] - Maximum requests per minute allowed - - budget_duration: Optional[str] - Duration for budget renewal (e.g., "30d" for 30 days) - - allowed_cache_controls: Optional[list] - List of allowed cache control options - - soft_budget: Optional[float] - Soft budget limit for alerting purposes + - user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated. + - user_email: Optional[str] - Specify a user email. + - password: Optional[str] - Specify a user password. + - user_alias: Optional[str] - A descriptive name for you to know who this user id refers to. + - teams: Optional[list] - specify a list of team id's a user belongs to. + - send_invite_email: Optional[bool] - Specify if an invite email should be sent. + - user_role: Optional[str] - Specify a user role - "proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer", "team", "customer". Info about each role here: `https://github.com/BerriAI/litellm/litellm/proxy/_types.py#L20` + - max_budget: Optional[float] - Specify max budget for a given user. + - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). + - models: Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models) + - tpm_limit: Optional[int] - Specify tpm limit for a given user (Tokens per minute) + - rpm_limit: Optional[int] - Specify rpm limit for a given user (Requests per minute) + - auto_create_key: bool - Default=True. Flag used for returning a key as part of the /user/new response + - aliases: Optional[dict] - Model aliases for the user - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases) + - config: Optional[dict] - [DEPRECATED PARAM] User-specific config. + - allowed_cache_controls: Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request- + - blocked: Optional[bool] - [Not Implemented Yet] Whether the user is blocked. + - guardrails: Optional[List[str]] - [Not Implemented Yet] List of active guardrails for the user + - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. + - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } + - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. + - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. + - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) + - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) + - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) + - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). + - team_id: Optional[str] - [DEPRECATED PARAM] The team id of the user. Default is None. + - duration: Optional[str] - [NOT IMPLEMENTED]. + - key_alias: Optional[str] - [NOT IMPLEMENTED]. + ``` """ from litellm.proxy.proxy_server import prisma_client diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c2de82ce7..e4493a28c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -83,6 +83,13 @@ async def generate_key_fn( # noqa: PLR0915 - model_max_budget: Optional[dict] - key-specific model budget in USD. Example - {"text-davinci-002": 0.5, "gpt-3.5-turbo": 0.5}. IF null or {} then no model specific budget. - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit. - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit. + - allowed_cache_controls: Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request + - blocked: Optional[bool] - Whether the key is blocked. + - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute) + - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute) + - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached. + - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). + Examples: 1. Allow users to turn on/off pii masking @@ -349,6 +356,8 @@ async def update_key_fn( - send_invite_email: Optional[bool] - Send invite email to user_id - guardrails: Optional[List[str]] - List of active guardrails for the key - blocked: Optional[bool] - Whether the key is blocked + - aliases: Optional[dict] - Model aliases for the key - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases) + - config: Optional[dict] - [DEPRECATED PARAM] Key-specific config. Example: ```bash diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index 5f58c4231..81d135097 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -5,6 +5,7 @@ Endpoints for /organization operations /organization/update /organization/delete /organization/info +/organization/list """ #### ORGANIZATION MANAGEMENT #### @@ -55,15 +56,23 @@ async def new_organization( # Parameters - - `organization_alias`: *str* = The name of the organization. - - `models`: *List* = The models the organization has access to. - - `budget_id`: *Optional[str]* = The id for a budget (tpm/rpm/max budget) for the organization. + - organization_alias: *str* - The name of the organization. + - models: *List* - The models the organization has access to. + - budget_id: *Optional[str]* - The id for a budget (tpm/rpm/max budget) for the organization. ### IF NO BUDGET ID - CREATE ONE WITH THESE PARAMS ### - - `max_budget`: *Optional[float]* = Max budget for org - - `tpm_limit`: *Optional[int]* = Max tpm limit for org - - `rpm_limit`: *Optional[int]* = Max rpm limit for org - - `model_max_budget`: *Optional[dict]* = Max budget for a specific model - - `budget_duration`: *Optional[str]* = Frequency of reseting org budget + - max_budget: *Optional[float]* - Max budget for org + - tpm_limit: *Optional[int]* - Max tpm limit for org + - rpm_limit: *Optional[int]* - Max rpm limit for org + - max_parallel_requests: *Optional[int]* - [Not Implemented Yet] Max parallel requests for org + - soft_budget: *Optional[float]* - [Not Implemented Yet] Get a slack alert when this soft budget is reached. Don't block requests. + - model_max_budget: *Optional[dict]* - Max budget for a specific model + - budget_duration: *Optional[str]* - Frequency of reseting org budget + - metadata: *Optional[dict]* - Metadata for team, store information for team. Example metadata - {"extra_info": "some info"} + - blocked: *bool* - Flag indicating if the org is blocked or not - will stop all calls from keys with this org_id. + - tags: *Optional[List[str]]* - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). + - organization_id: *Optional[str]* - The organization id of the team. Default is None. Create via `/organization/new`. + - model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias) + Case 1: Create new org **without** a budget_id @@ -185,7 +194,7 @@ async def new_organization( ) async def update_organization(): """[TODO] Not Implemented yet. Let us know if you need this - https://github.com/BerriAI/litellm/issues""" - pass + raise NotImplementedError("Not Implemented Yet") @router.post( @@ -195,7 +204,7 @@ async def update_organization(): ) async def delete_organization(): """[TODO] Not Implemented yet. Let us know if you need this - https://github.com/BerriAI/litellm/issues""" - pass + raise NotImplementedError("Not Implemented Yet") @router.get( @@ -204,38 +213,38 @@ async def delete_organization(): dependencies=[Depends(user_api_key_auth)], ) async def list_organization( - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): - """ + """ ``` curl --location --request GET 'http://0.0.0.0:4000/organization/list' \ --header 'Authorization: Bearer sk-1234' ``` """ - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - raise HTTPException(status_code=500, detail={"error": "No db connected"}) - - if ( - user_api_key_dict.user_role is None - or user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN - ): - raise HTTPException( - status_code=401, - detail={ - "error": f"Only admins can list orgs. Your role is = {user_api_key_dict.user_role}" - }, - ) - if prisma_client is None: - raise HTTPException( - status_code=400, - detail={"error": CommonProxyErrors.db_not_connected_error.value}, - ) - response= await prisma_client.db.litellm_organizationtable.find_many() + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + if ( + user_api_key_dict.user_role is None + or user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN + ): + raise HTTPException( + status_code=401, + detail={ + "error": f"Only admins can list orgs. Your role is = {user_api_key_dict.user_role}" + }, + ) + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + response = await prisma_client.db.litellm_organizationtable.find_many() + + return response - return response - @router.post( "/organization/info", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 251fa648e..dc1ec444d 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1,3 +1,14 @@ +""" +TEAM MANAGEMENT + +All /team management endpoints + +/team/new +/team/info +/team/update +/team/delete +""" + import asyncio import copy import json @@ -121,6 +132,10 @@ async def new_team( # noqa: PLR0915 - budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) - models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed. - blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id. + - members: Optional[List] - Control team members via `/team/member/add` and `/team/member/delete`. + - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). + - organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`. + - model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias) Returns: - team_id: (str) Unique team id - used for tracking spend across multiple keys for same team id. @@ -353,6 +368,8 @@ async def update_team( - budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) - models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed. - blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id. + - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). + - organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`. Example - update team TPM Limit diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e495f3490..74bf398e7 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3127,6 +3127,7 @@ def _get_docs_url() -> Optional[str]: # default to "/" return "/" + def handle_exception_on_proxy(e: Exception) -> ProxyException: """ Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible @@ -3148,4 +3149,3 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException: param=getattr(e, "param", "None"), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - diff --git a/tests/documentation_tests/test_api_docs.py b/tests/documentation_tests/test_api_docs.py new file mode 100644 index 000000000..407010dcc --- /dev/null +++ b/tests/documentation_tests/test_api_docs.py @@ -0,0 +1,206 @@ +import ast +from typing import List, Dict, Set, Optional +import os +from dataclasses import dataclass +import argparse +import re +import sys + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm + + +@dataclass +class FunctionInfo: + """Store function information.""" + + name: str + docstring: Optional[str] + parameters: Set[str] + file_path: str + line_number: int + + +class FastAPIDocVisitor(ast.NodeVisitor): + """AST visitor to find FastAPI endpoint functions.""" + + def __init__(self, target_functions: Set[str]): + self.target_functions = target_functions + self.functions: Dict[str, FunctionInfo] = {} + self.current_file = "" + + def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + """Visit function definitions (both async and sync) and collect info if they match target functions.""" + if node.name in self.target_functions: + # Extract docstring + docstring = ast.get_docstring(node) + + # Extract parameters + parameters = set() + for arg in node.args.args: + if arg.annotation is not None: + # Get the parameter type from annotation + if isinstance(arg.annotation, ast.Name): + parameters.add((arg.arg, arg.annotation.id)) + elif isinstance(arg.annotation, ast.Subscript): + if isinstance(arg.annotation.value, ast.Name): + parameters.add((arg.arg, arg.annotation.value.id)) + + self.functions[node.name] = FunctionInfo( + name=node.name, + docstring=docstring, + parameters=parameters, + file_path=self.current_file, + line_number=node.lineno, + ) + + # Also need to add this to handle async functions + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Handle async functions by delegating to the regular function visitor.""" + return self.visit_FunctionDef(node) + + +def find_functions_in_file( + file_path: str, target_functions: Set[str] +) -> Dict[str, FunctionInfo]: + """Find target functions in a Python file using AST.""" + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + visitor = FastAPIDocVisitor(target_functions) + visitor.current_file = file_path + tree = ast.parse(content) + visitor.visit(tree) + return visitor.functions + + except Exception as e: + print(f"Error parsing {file_path}: {str(e)}") + return {} + + +def extract_docstring_params(docstring: Optional[str]) -> Set[str]: + """Extract parameter names from docstring.""" + if not docstring: + return set() + + params = set() + # Match parameters in format: + # - parameter_name: description + # or + # parameter_name: description + param_pattern = r"-?\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*(?:\([^)]*\))?\s*:" + + for match in re.finditer(param_pattern, docstring): + params.add(match.group(1)) + + return params + + +def analyze_function(func_info: FunctionInfo) -> Dict: + """Analyze function documentation and return validation results.""" + + docstring_params = extract_docstring_params(func_info.docstring) + + print(f"func_info.parameters: {func_info.parameters}") + pydantic_params = set() + + for name, type_name in func_info.parameters: + if type_name.endswith("Request") or type_name.endswith("Response"): + pydantic_model = getattr(litellm.proxy._types, type_name, None) + if pydantic_model is not None: + for param in pydantic_model.model_fields.keys(): + pydantic_params.add(param) + + print(f"pydantic_params: {pydantic_params}") + + missing_params = pydantic_params - docstring_params + + return { + "function": func_info.name, + "file_path": func_info.file_path, + "line_number": func_info.line_number, + "has_docstring": bool(func_info.docstring), + "pydantic_params": list(pydantic_params), + "documented_params": list(docstring_params), + "missing_params": list(missing_params), + "is_valid": len(missing_params) == 0, + } + + +def print_validation_results(results: Dict) -> None: + """Print validation results in a readable format.""" + print(f"\nChecking function: {results['function']}") + print(f"File: {results['file_path']}:{results['line_number']}") + print("-" * 50) + + if not results["has_docstring"]: + print("❌ No docstring found!") + return + + if not results["pydantic_params"]: + print("ℹ️ No Pydantic input models found.") + return + + if results["is_valid"]: + print("✅ All Pydantic parameters are documented!") + else: + print("❌ Missing documentation for parameters:") + for param in sorted(results["missing_params"]): + print(f" - {param}") + + +def main(): + function_names = [ + "new_end_user", + "end_user_info", + "update_end_user", + "delete_end_user", + "generate_key_fn", + "info_key_fn", + "update_key_fn", + "delete_key_fn", + "new_user", + "new_team", + "team_info", + "update_team", + "delete_team", + "new_organization", + "update_organization", + "delete_organization", + "list_organization", + "user_update", + ] + directory = "../../litellm/proxy/management_endpoints" # LOCAL + # directory = "./litellm/proxy/management_endpoints" + + # Convert function names to set for faster lookup + target_functions = set(function_names) + found_functions: Dict[str, FunctionInfo] = {} + + # Walk through directory + for root, _, files in os.walk(directory): + for file in files: + if file.endswith(".py"): + file_path = os.path.join(root, file) + found = find_functions_in_file(file_path, target_functions) + found_functions.update(found) + + # Analyze and output results + for func_name in function_names: + if func_name in found_functions: + result = analyze_function(found_functions[func_name]) + if not result["is_valid"]: + raise Exception(print_validation_results(result)) + # results.append(result) + # print_validation_results(result) + + # # Exit with error code if any validation failed + # if any(not r["is_valid"] for r in results): + # exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index 8ad773d63..e6f8ca541 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -1018,7 +1018,7 @@ def test_generate_and_call_with_expired_key(prisma_client): # use generated key to auth in result = await user_api_key_auth(request=request, api_key=bearer_token) print("result from user auth with new key", result) - pytest.fail(f"This should have failed!. IT's an expired key") + pytest.fail("This should have failed!. It's an expired key") asyncio.run(test()) except Exception as e: From 746881485f678bf4bfbf0b2ec55b054cd01e114f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 21 Nov 2024 04:38:04 +0530 Subject: [PATCH 026/113] =?UTF-8?q?bump:=20version=201.52.11=20=E2=86=92?= =?UTF-8?q?=201.52.12?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c57bc5de..3e69461ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.52.11" +version = "1.52.12" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.52.11" +version = "1.52.12" version_files = [ "pyproject.toml:^version" ] From 0b0253f7add9700b78310253d64dffddb25895bb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 21 Nov 2024 05:16:58 +0530 Subject: [PATCH 027/113] build: update ui build --- litellm/proxy/_experimental/out/404.html | 2 +- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 .../static/chunks/131-3d2257b0ff5aadb2.js | 8 ---- .../static/chunks/131-4ee1d633e8928742.js | 8 ++++ .../static/chunks/626-0c564a21577c9c53.js | 13 ++++++ .../static/chunks/626-4e8df4039ecf4386.js | 13 ------ .../chunks/app/layout-61827b157521da1b.js | 1 - .../chunks/app/layout-77825730d130b292.js | 1 + .../app/model_hub/page-104cada6b5e5b14c.js | 1 - .../app/model_hub/page-748a83a8e772a56b.js | 2 +- ...be58b9d19c.js => page-884a15d08f8be397.js} | 2 +- ...46118fabbb.js => page-413af091866cb902.js} | 2 +- ...b53edf.js => main-app-096338c8e1915716.js} | 2 +- ...0c46e0b.js => webpack-a13477d480030cb3.js} | 2 +- ...56a1984d35914.css => 8fbba1b67a4788fc.css} | 4 +- .../static/development/_buildManifest.js | 1 - .../static/media/05a31a2ca4975f99-s.woff2 | Bin 0 -> 10496 bytes .../static/media/26a46d62cd723877-s.woff2 | Bin 18820 -> 0 bytes .../static/media/513657b02c5c193f-s.woff2 | Bin 0 -> 17612 bytes .../static/media/51ed15f9841b9f9d-s.woff2 | Bin 0 -> 22524 bytes .../static/media/55c55f0601d81cf3-s.woff2 | Bin 25908 -> 0 bytes .../static/media/581909926a08bbc8-s.woff2 | Bin 19072 -> 0 bytes .../static/media/6d93bde91c0c2823-s.woff2 | Bin 74316 -> 0 bytes .../static/media/97e0cb1ae144a2a9-s.woff2 | Bin 11220 -> 0 bytes .../static/media/a34f9d1faa5f3315-s.p.woff2 | Bin 48556 -> 0 bytes .../static/media/c9a5bc6a7c948fb0-s.p.woff2 | Bin 0 -> 46552 bytes .../static/media/d6b16ce4a6175f26-s.woff2 | Bin 0 -> 80044 bytes .../static/media/df0a9ae256c0569c-s.woff2 | Bin 10280 -> 0 bytes .../static/media/ec159349637c90ad-s.woff2 | Bin 0 -> 27316 bytes .../static/media/fd4db3eb5472fc27-s.woff2 | Bin 0 -> 12768 bytes .../app/layout.ad2650a809509e80.hot-update.js | 22 --------- .../app/page.ad2650a809509e80.hot-update.js | 42 ------------------ litellm/proxy/_experimental/out/index.html | 2 +- litellm/proxy/_experimental/out/index.txt | 4 +- .../proxy/_experimental/out/model_hub.html | 2 +- litellm/proxy/_experimental/out/model_hub.txt | 4 +- .../proxy/_experimental/out/onboarding.html | 2 +- .../proxy/_experimental/out/onboarding.txt | 4 +- ui/litellm-dashboard/out/404.html | 2 +- .../static/chunks/131-3d2257b0ff5aadb2.js | 8 ---- .../static/chunks/626-4e8df4039ecf4386.js | 13 ------ .../chunks/app/layout-61827b157521da1b.js | 1 - .../app/onboarding/page-bad6cfbe58b9d19c.js | 1 - .../chunks/app/page-3b1ed846118fabbb.js | 1 - .../chunks/main-app-9b4fb13a7db53edf.js | 1 - .../static/chunks/webpack-e8ad0a25b0c46e0b.js | 1 - .../out/_next/static/css/00256a1984d35914.css | 5 --- .../dcp3YN3z2izmIjGczDqPp/_buildManifest.js | 1 - .../dcp3YN3z2izmIjGczDqPp/_ssgManifest.js | 1 - .../static/development/_buildManifest.js | 1 - .../static/media/26a46d62cd723877-s.woff2 | Bin 18820 -> 0 bytes .../static/media/55c55f0601d81cf3-s.woff2 | Bin 25908 -> 0 bytes .../static/media/581909926a08bbc8-s.woff2 | Bin 19072 -> 0 bytes .../static/media/6d93bde91c0c2823-s.woff2 | Bin 74316 -> 0 bytes .../static/media/97e0cb1ae144a2a9-s.woff2 | Bin 11220 -> 0 bytes .../static/media/a34f9d1faa5f3315-s.p.woff2 | Bin 48556 -> 0 bytes .../static/media/df0a9ae256c0569c-s.woff2 | Bin 10280 -> 0 bytes .../app/layout.ad2650a809509e80.hot-update.js | 22 --------- .../app/page.ad2650a809509e80.hot-update.js | 42 ------------------ ui/litellm-dashboard/out/index.html | 2 +- ui/litellm-dashboard/out/index.txt | 4 +- ui/litellm-dashboard/out/model_hub.html | 2 +- ui/litellm-dashboard/out/model_hub.txt | 4 +- ui/litellm-dashboard/out/onboarding.html | 2 +- ui/litellm-dashboard/out/onboarding.txt | 4 +- 66 files changed, 48 insertions(+), 214 deletions(-) rename litellm/proxy/_experimental/out/_next/static/{dcp3YN3z2izmIjGczDqPp => 4u3imMIH2UVoP8L-yPCjs}/_buildManifest.js (100%) rename litellm/proxy/_experimental/out/_next/static/{dcp3YN3z2izmIjGczDqPp => 4u3imMIH2UVoP8L-yPCjs}/_ssgManifest.js (100%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/131-3d2257b0ff5aadb2.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/131-4ee1d633e8928742.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/626-0c564a21577c9c53.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/626-4e8df4039ecf4386.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/layout-61827b157521da1b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/layout-77825730d130b292.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-104cada6b5e5b14c.js rename ui/litellm-dashboard/out/_next/static/chunks/app/model_hub/page-104cada6b5e5b14c.js => litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-748a83a8e772a56b.js (97%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/{page-bad6cfbe58b9d19c.js => page-884a15d08f8be397.js} (94%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/{page-3b1ed846118fabbb.js => page-413af091866cb902.js} (55%) rename litellm/proxy/_experimental/out/_next/static/chunks/{main-app-9b4fb13a7db53edf.js => main-app-096338c8e1915716.js} (54%) rename litellm/proxy/_experimental/out/_next/static/chunks/{webpack-e8ad0a25b0c46e0b.js => webpack-a13477d480030cb3.js} (98%) rename litellm/proxy/_experimental/out/_next/static/css/{00256a1984d35914.css => 8fbba1b67a4788fc.css} (99%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/development/_buildManifest.js create mode 100644 litellm/proxy/_experimental/out/_next/static/media/05a31a2ca4975f99-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/26a46d62cd723877-s.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/513657b02c5c193f-s.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/51ed15f9841b9f9d-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/55c55f0601d81cf3-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/581909926a08bbc8-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/6d93bde91c0c2823-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/97e0cb1ae144a2a9-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/a34f9d1faa5f3315-s.p.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/d6b16ce4a6175f26-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/df0a9ae256c0569c-s.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/ec159349637c90ad-s.woff2 create mode 100644 litellm/proxy/_experimental/out/_next/static/media/fd4db3eb5472fc27-s.woff2 delete mode 100644 litellm/proxy/_experimental/out/_next/static/webpack/app/layout.ad2650a809509e80.hot-update.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/webpack/app/page.ad2650a809509e80.hot-update.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/131-3d2257b0ff5aadb2.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/626-4e8df4039ecf4386.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/layout-61827b157521da1b.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/onboarding/page-bad6cfbe58b9d19c.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/page-3b1ed846118fabbb.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/main-app-9b4fb13a7db53edf.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/webpack-e8ad0a25b0c46e0b.js delete mode 100644 ui/litellm-dashboard/out/_next/static/css/00256a1984d35914.css delete mode 100644 ui/litellm-dashboard/out/_next/static/dcp3YN3z2izmIjGczDqPp/_buildManifest.js delete mode 100644 ui/litellm-dashboard/out/_next/static/dcp3YN3z2izmIjGczDqPp/_ssgManifest.js delete mode 100644 ui/litellm-dashboard/out/_next/static/development/_buildManifest.js delete mode 100644 ui/litellm-dashboard/out/_next/static/media/26a46d62cd723877-s.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/media/55c55f0601d81cf3-s.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/media/581909926a08bbc8-s.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/media/6d93bde91c0c2823-s.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/media/97e0cb1ae144a2a9-s.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/media/a34f9d1faa5f3315-s.p.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/media/df0a9ae256c0569c-s.woff2 delete mode 100644 ui/litellm-dashboard/out/_next/static/webpack/app/layout.ad2650a809509e80.hot-update.js delete mode 100644 ui/litellm-dashboard/out/_next/static/webpack/app/page.ad2650a809509e80.hot-update.js diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index efcf1893d..09dcdd244 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/dcp3YN3z2izmIjGczDqPp/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/4u3imMIH2UVoP8L-yPCjs/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/dcp3YN3z2izmIjGczDqPp/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/4u3imMIH2UVoP8L-yPCjs/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/dcp3YN3z2izmIjGczDqPp/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/4u3imMIH2UVoP8L-yPCjs/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/dcp3YN3z2izmIjGczDqPp/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/4u3imMIH2UVoP8L-yPCjs/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/131-3d2257b0ff5aadb2.js b/litellm/proxy/_experimental/out/_next/static/chunks/131-3d2257b0ff5aadb2.js deleted file mode 100644 index 51181e75a..000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/131-3d2257b0ff5aadb2.js +++ /dev/null @@ -1,8 +0,0 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[131],{84174:function(e,t,n){n.d(t,{Z:function(){return s}});var a=n(14749),r=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M832 64H296c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h496v688c0 4.4 3.6 8 8 8h56c4.4 0 8-3.6 8-8V96c0-17.7-14.3-32-32-32zM704 192H192c-17.7 0-32 14.3-32 32v530.7c0 8.5 3.4 16.6 9.4 22.6l173.3 173.3c2.2 2.2 4.7 4 7.4 5.5v1.9h4.2c3.5 1.3 7.2 2 11 2H704c17.7 0 32-14.3 32-32V224c0-17.7-14.3-32-32-32zM350 856.2L263.9 770H350v86.2zM664 888H414V746c0-22.1-17.9-40-40-40H232V264h432v624z"}}]},name:"copy",theme:"outlined"},o=n(60688),s=r.forwardRef(function(e,t){return r.createElement(o.Z,(0,a.Z)({},e,{ref:t,icon:i}))})},50459:function(e,t,n){n.d(t,{Z:function(){return s}});var a=n(14749),r=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M765.7 486.8L314.9 134.7A7.97 7.97 0 00302 141v77.3c0 4.9 2.3 9.6 6.1 12.6l360 281.1-360 281.1c-3.9 3-6.1 7.7-6.1 12.6V883c0 6.7 7.7 10.4 12.9 6.3l450.8-352.1a31.96 31.96 0 000-50.4z"}}]},name:"right",theme:"outlined"},o=n(60688),s=r.forwardRef(function(e,t){return r.createElement(o.Z,(0,a.Z)({},e,{ref:t,icon:i}))})},92836:function(e,t,n){n.d(t,{Z:function(){return p}});var a=n(69703),r=n(80991),i=n(2898),o=n(99250),s=n(65492),l=n(2265),c=n(41608),d=n(50027);n(18174),n(21871),n(41213);let u=(0,s.fn)("Tab"),p=l.forwardRef((e,t)=>{let{icon:n,className:p,children:g}=e,m=(0,a._T)(e,["icon","className","children"]),b=(0,l.useContext)(c.O),f=(0,l.useContext)(d.Z);return l.createElement(r.O,Object.assign({ref:t,className:(0,o.q)(u("root"),"flex whitespace-nowrap truncate max-w-xs outline-none focus:ring-0 text-tremor-default transition duration-100",f?(0,s.bM)(f,i.K.text).selectTextColor:"solid"===b?"ui-selected:text-tremor-content-emphasis dark:ui-selected:text-dark-tremor-content-emphasis":"ui-selected:text-tremor-brand dark:ui-selected:text-dark-tremor-brand",function(e,t){switch(e){case"line":return(0,o.q)("ui-selected:border-b-2 hover:border-b-2 border-transparent transition duration-100 -mb-px px-2 py-2","hover:border-tremor-content hover:text-tremor-content-emphasis text-tremor-content","dark:hover:border-dark-tremor-content-emphasis dark:hover:text-dark-tremor-content-emphasis dark:text-dark-tremor-content",t?(0,s.bM)(t,i.K.border).selectBorderColor:"ui-selected:border-tremor-brand dark:ui-selected:border-dark-tremor-brand");case"solid":return(0,o.q)("border-transparent border rounded-tremor-small px-2.5 py-1","ui-selected:border-tremor-border ui-selected:bg-tremor-background ui-selected:shadow-tremor-input hover:text-tremor-content-emphasis ui-selected:text-tremor-brand","dark:ui-selected:border-dark-tremor-border dark:ui-selected:bg-dark-tremor-background dark:ui-selected:shadow-dark-tremor-input dark:hover:text-dark-tremor-content-emphasis dark:ui-selected:text-dark-tremor-brand",t?(0,s.bM)(t,i.K.text).selectTextColor:"text-tremor-content dark:text-dark-tremor-content")}}(b,f),p)},m),n?l.createElement(n,{className:(0,o.q)(u("icon"),"flex-none h-5 w-5",g?"mr-2":"")}):null,g?l.createElement("span",null,g):null)});p.displayName="Tab"},26734:function(e,t,n){n.d(t,{Z:function(){return c}});var a=n(69703),r=n(80991),i=n(99250),o=n(65492),s=n(2265);let l=(0,o.fn)("TabGroup"),c=s.forwardRef((e,t)=>{let{defaultIndex:n,index:o,onIndexChange:c,children:d,className:u}=e,p=(0,a._T)(e,["defaultIndex","index","onIndexChange","children","className"]);return s.createElement(r.O.Group,Object.assign({as:"div",ref:t,defaultIndex:n,selectedIndex:o,onChange:c,className:(0,i.q)(l("root"),"w-full",u)},p),d)});c.displayName="TabGroup"},41608:function(e,t,n){n.d(t,{O:function(){return c},Z:function(){return u}});var a=n(69703),r=n(2265),i=n(50027);n(18174),n(21871),n(41213);var o=n(80991),s=n(99250);let l=(0,n(65492).fn)("TabList"),c=(0,r.createContext)("line"),d={line:(0,s.q)("flex border-b space-x-4","border-tremor-border","dark:border-dark-tremor-border"),solid:(0,s.q)("inline-flex p-0.5 rounded-tremor-default space-x-1.5","bg-tremor-background-subtle","dark:bg-dark-tremor-background-subtle")},u=r.forwardRef((e,t)=>{let{color:n,variant:u="line",children:p,className:g}=e,m=(0,a._T)(e,["color","variant","children","className"]);return r.createElement(o.O.List,Object.assign({ref:t,className:(0,s.q)(l("root"),"justify-start overflow-x-clip",d[u],g)},m),r.createElement(c.Provider,{value:u},r.createElement(i.Z.Provider,{value:n},p)))});u.displayName="TabList"},32126:function(e,t,n){n.d(t,{Z:function(){return d}});var a=n(69703);n(50027);var r=n(18174);n(21871);var i=n(41213),o=n(99250),s=n(65492),l=n(2265);let c=(0,s.fn)("TabPanel"),d=l.forwardRef((e,t)=>{let{children:n,className:s}=e,d=(0,a._T)(e,["children","className"]),{selectedValue:u}=(0,l.useContext)(i.Z),p=u===(0,l.useContext)(r.Z);return l.createElement("div",Object.assign({ref:t,className:(0,o.q)(c("root"),"w-full mt-2",p?"":"hidden",s),"aria-selected":p?"true":"false"},d),n)});d.displayName="TabPanel"},23682:function(e,t,n){n.d(t,{Z:function(){return u}});var a=n(69703),r=n(80991);n(50027);var i=n(18174);n(21871);var o=n(41213),s=n(99250),l=n(65492),c=n(2265);let d=(0,l.fn)("TabPanels"),u=c.forwardRef((e,t)=>{let{children:n,className:l}=e,u=(0,a._T)(e,["children","className"]);return c.createElement(r.O.Panels,Object.assign({as:"div",ref:t,className:(0,s.q)(d("root"),"w-full",l)},u),e=>{let{selectedIndex:t}=e;return c.createElement(o.Z.Provider,{value:{selectedValue:t}},c.Children.map(n,(e,t)=>c.createElement(i.Z.Provider,{value:t},e)))})});u.displayName="TabPanels"},50027:function(e,t,n){n.d(t,{Z:function(){return i}});var a=n(2265),r=n(54942);n(99250);let i=(0,a.createContext)(r.fr.Blue)},18174:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(2265).createContext)(0)},21871:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(2265).createContext)(void 0)},41213:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(2265).createContext)({selectedValue:void 0,handleValueChange:void 0})},21467:function(e,t,n){n.d(t,{i:function(){return s}});var a=n(2265),r=n(44329),i=n(54165),o=n(57499);function s(e){return t=>a.createElement(i.ZP,{theme:{token:{motion:!1,zIndexPopupBase:0}}},a.createElement(e,Object.assign({},t)))}t.Z=(e,t,n,i)=>s(s=>{let{prefixCls:l,style:c}=s,d=a.useRef(null),[u,p]=a.useState(0),[g,m]=a.useState(0),[b,f]=(0,r.Z)(!1,{value:s.open}),{getPrefixCls:E}=a.useContext(o.E_),h=E(t||"select",l);a.useEffect(()=>{if(f(!0),"undefined"!=typeof ResizeObserver){let e=new ResizeObserver(e=>{let t=e[0].target;p(t.offsetHeight+8),m(t.offsetWidth)}),t=setInterval(()=>{var a;let r=n?".".concat(n(h)):".".concat(h,"-dropdown"),i=null===(a=d.current)||void 0===a?void 0:a.querySelector(r);i&&(clearInterval(t),e.observe(i))},10);return()=>{clearInterval(t),e.disconnect()}}},[]);let S=Object.assign(Object.assign({},s),{style:Object.assign(Object.assign({},c),{margin:0}),open:b,visible:b,getPopupContainer:()=>d.current});return i&&(S=i(S)),a.createElement("div",{ref:d,style:{paddingBottom:u,position:"relative",minWidth:g}},a.createElement(e,Object.assign({},S)))})},99129:function(e,t,n){let a;n.d(t,{Z:function(){return eY}});var r=n(63787),i=n(2265),o=n(37274),s=n(57499),l=n(54165),c=n(99537),d=n(77136),u=n(20653),p=n(40388),g=n(16480),m=n.n(g),b=n(51761),f=n(47387),E=n(70595),h=n(24750),S=n(89211),y=n(13565),T=n(51350),A=e=>{let{type:t,children:n,prefixCls:a,buttonProps:r,close:o,autoFocus:s,emitEvent:l,isSilent:c,quitOnNullishReturnValue:d,actionFn:u}=e,p=i.useRef(!1),g=i.useRef(null),[m,b]=(0,S.Z)(!1),f=function(){null==o||o.apply(void 0,arguments)};i.useEffect(()=>{let e=null;return s&&(e=setTimeout(()=>{var e;null===(e=g.current)||void 0===e||e.focus()})),()=>{e&&clearTimeout(e)}},[]);let E=e=>{e&&e.then&&(b(!0),e.then(function(){b(!1,!0),f.apply(void 0,arguments),p.current=!1},e=>{if(b(!1,!0),p.current=!1,null==c||!c())return Promise.reject(e)}))};return i.createElement(y.ZP,Object.assign({},(0,T.nx)(t),{onClick:e=>{let t;if(!p.current){if(p.current=!0,!u){f();return}if(l){var n;if(t=u(e),d&&!((n=t)&&n.then)){p.current=!1,f(e);return}}else if(u.length)t=u(o),p.current=!1;else if(!(t=u())){f();return}E(t)}},loading:m,prefixCls:a},r,{ref:g}),n)};let R=i.createContext({}),{Provider:I}=R;var N=()=>{let{autoFocusButton:e,cancelButtonProps:t,cancelTextLocale:n,isSilent:a,mergedOkCancel:r,rootPrefixCls:o,close:s,onCancel:l,onConfirm:c}=(0,i.useContext)(R);return r?i.createElement(A,{isSilent:a,actionFn:l,close:function(){null==s||s.apply(void 0,arguments),null==c||c(!1)},autoFocus:"cancel"===e,buttonProps:t,prefixCls:"".concat(o,"-btn")},n):null},_=()=>{let{autoFocusButton:e,close:t,isSilent:n,okButtonProps:a,rootPrefixCls:r,okTextLocale:o,okType:s,onConfirm:l,onOk:c}=(0,i.useContext)(R);return i.createElement(A,{isSilent:n,type:s||"primary",actionFn:c,close:function(){null==t||t.apply(void 0,arguments),null==l||l(!0)},autoFocus:"ok"===e,buttonProps:a,prefixCls:"".concat(r,"-btn")},o)},v=n(81303),w=n(14749),k=n(80406),C=n(88804),O=i.createContext({}),x=n(5239),L=n(31506),D=n(91010),P=n(4295),M=n(72480);function F(e,t,n){var a=t;return!a&&n&&(a="".concat(e,"-").concat(n)),a}function U(e,t){var n=e["page".concat(t?"Y":"X","Offset")],a="scroll".concat(t?"Top":"Left");if("number"!=typeof n){var r=e.document;"number"!=typeof(n=r.documentElement[a])&&(n=r.body[a])}return n}var B=n(49367),G=n(74084),$=i.memo(function(e){return e.children},function(e,t){return!t.shouldUpdate}),H={width:0,height:0,overflow:"hidden",outline:"none"},z=i.forwardRef(function(e,t){var n,a,r,o=e.prefixCls,s=e.className,l=e.style,c=e.title,d=e.ariaId,u=e.footer,p=e.closable,g=e.closeIcon,b=e.onClose,f=e.children,E=e.bodyStyle,h=e.bodyProps,S=e.modalRender,y=e.onMouseDown,T=e.onMouseUp,A=e.holderRef,R=e.visible,I=e.forceRender,N=e.width,_=e.height,v=e.classNames,k=e.styles,C=i.useContext(O).panel,L=(0,G.x1)(A,C),D=(0,i.useRef)(),P=(0,i.useRef)();i.useImperativeHandle(t,function(){return{focus:function(){var e;null===(e=D.current)||void 0===e||e.focus()},changeActive:function(e){var t=document.activeElement;e&&t===P.current?D.current.focus():e||t!==D.current||P.current.focus()}}});var M={};void 0!==N&&(M.width=N),void 0!==_&&(M.height=_),u&&(n=i.createElement("div",{className:m()("".concat(o,"-footer"),null==v?void 0:v.footer),style:(0,x.Z)({},null==k?void 0:k.footer)},u)),c&&(a=i.createElement("div",{className:m()("".concat(o,"-header"),null==v?void 0:v.header),style:(0,x.Z)({},null==k?void 0:k.header)},i.createElement("div",{className:"".concat(o,"-title"),id:d},c))),p&&(r=i.createElement("button",{type:"button",onClick:b,"aria-label":"Close",className:"".concat(o,"-close")},g||i.createElement("span",{className:"".concat(o,"-close-x")})));var F=i.createElement("div",{className:m()("".concat(o,"-content"),null==v?void 0:v.content),style:null==k?void 0:k.content},r,a,i.createElement("div",(0,w.Z)({className:m()("".concat(o,"-body"),null==v?void 0:v.body),style:(0,x.Z)((0,x.Z)({},E),null==k?void 0:k.body)},h),f),n);return i.createElement("div",{key:"dialog-element",role:"dialog","aria-labelledby":c?d:null,"aria-modal":"true",ref:L,style:(0,x.Z)((0,x.Z)({},l),M),className:m()(o,s),onMouseDown:y,onMouseUp:T},i.createElement("div",{tabIndex:0,ref:D,style:H,"aria-hidden":"true"}),i.createElement($,{shouldUpdate:R||I},S?S(F):F),i.createElement("div",{tabIndex:0,ref:P,style:H,"aria-hidden":"true"}))}),j=i.forwardRef(function(e,t){var n=e.prefixCls,a=e.title,r=e.style,o=e.className,s=e.visible,l=e.forceRender,c=e.destroyOnClose,d=e.motionName,u=e.ariaId,p=e.onVisibleChanged,g=e.mousePosition,b=(0,i.useRef)(),f=i.useState(),E=(0,k.Z)(f,2),h=E[0],S=E[1],y={};function T(){var e,t,n,a,r,i=(n={left:(t=(e=b.current).getBoundingClientRect()).left,top:t.top},r=(a=e.ownerDocument).defaultView||a.parentWindow,n.left+=U(r),n.top+=U(r,!0),n);S(g?"".concat(g.x-i.left,"px ").concat(g.y-i.top,"px"):"")}return h&&(y.transformOrigin=h),i.createElement(B.ZP,{visible:s,onVisibleChanged:p,onAppearPrepare:T,onEnterPrepare:T,forceRender:l,motionName:d,removeOnLeave:c,ref:b},function(s,l){var c=s.className,d=s.style;return i.createElement(z,(0,w.Z)({},e,{ref:t,title:a,ariaId:u,prefixCls:n,holderRef:l,style:(0,x.Z)((0,x.Z)((0,x.Z)({},d),r),y),className:m()(o,c)}))})});function V(e){var t=e.prefixCls,n=e.style,a=e.visible,r=e.maskProps,o=e.motionName,s=e.className;return i.createElement(B.ZP,{key:"mask",visible:a,motionName:o,leavedClassName:"".concat(t,"-mask-hidden")},function(e,a){var o=e.className,l=e.style;return i.createElement("div",(0,w.Z)({ref:a,style:(0,x.Z)((0,x.Z)({},l),n),className:m()("".concat(t,"-mask"),o,s)},r))})}function W(e){var t=e.prefixCls,n=void 0===t?"rc-dialog":t,a=e.zIndex,r=e.visible,o=void 0!==r&&r,s=e.keyboard,l=void 0===s||s,c=e.focusTriggerAfterClose,d=void 0===c||c,u=e.wrapStyle,p=e.wrapClassName,g=e.wrapProps,b=e.onClose,f=e.afterOpenChange,E=e.afterClose,h=e.transitionName,S=e.animation,y=e.closable,T=e.mask,A=void 0===T||T,R=e.maskTransitionName,I=e.maskAnimation,N=e.maskClosable,_=e.maskStyle,v=e.maskProps,C=e.rootClassName,O=e.classNames,U=e.styles,B=(0,i.useRef)(),G=(0,i.useRef)(),$=(0,i.useRef)(),H=i.useState(o),z=(0,k.Z)(H,2),W=z[0],q=z[1],Y=(0,D.Z)();function K(e){null==b||b(e)}var Z=(0,i.useRef)(!1),X=(0,i.useRef)(),Q=null;return(void 0===N||N)&&(Q=function(e){Z.current?Z.current=!1:G.current===e.target&&K(e)}),(0,i.useEffect)(function(){o&&(q(!0),(0,L.Z)(G.current,document.activeElement)||(B.current=document.activeElement))},[o]),(0,i.useEffect)(function(){return function(){clearTimeout(X.current)}},[]),i.createElement("div",(0,w.Z)({className:m()("".concat(n,"-root"),C)},(0,M.Z)(e,{data:!0})),i.createElement(V,{prefixCls:n,visible:A&&o,motionName:F(n,R,I),style:(0,x.Z)((0,x.Z)({zIndex:a},_),null==U?void 0:U.mask),maskProps:v,className:null==O?void 0:O.mask}),i.createElement("div",(0,w.Z)({tabIndex:-1,onKeyDown:function(e){if(l&&e.keyCode===P.Z.ESC){e.stopPropagation(),K(e);return}o&&e.keyCode===P.Z.TAB&&$.current.changeActive(!e.shiftKey)},className:m()("".concat(n,"-wrap"),p,null==O?void 0:O.wrapper),ref:G,onClick:Q,style:(0,x.Z)((0,x.Z)((0,x.Z)({zIndex:a},u),null==U?void 0:U.wrapper),{},{display:W?null:"none"})},g),i.createElement(j,(0,w.Z)({},e,{onMouseDown:function(){clearTimeout(X.current),Z.current=!0},onMouseUp:function(){X.current=setTimeout(function(){Z.current=!1})},ref:$,closable:void 0===y||y,ariaId:Y,prefixCls:n,visible:o&&W,onClose:K,onVisibleChanged:function(e){if(e)!function(){if(!(0,L.Z)(G.current,document.activeElement)){var e;null===(e=$.current)||void 0===e||e.focus()}}();else{if(q(!1),A&&B.current&&d){try{B.current.focus({preventScroll:!0})}catch(e){}B.current=null}W&&(null==E||E())}null==f||f(e)},motionName:F(n,h,S)}))))}j.displayName="Content",n(53850);var q=function(e){var t=e.visible,n=e.getContainer,a=e.forceRender,r=e.destroyOnClose,o=void 0!==r&&r,s=e.afterClose,l=e.panelRef,c=i.useState(t),d=(0,k.Z)(c,2),u=d[0],p=d[1],g=i.useMemo(function(){return{panel:l}},[l]);return(i.useEffect(function(){t&&p(!0)},[t]),a||!o||u)?i.createElement(O.Provider,{value:g},i.createElement(C.Z,{open:t||a||u,autoDestroy:!1,getContainer:n,autoLock:t||u},i.createElement(W,(0,w.Z)({},e,{destroyOnClose:o,afterClose:function(){null==s||s(),p(!1)}})))):null};q.displayName="Dialog";var Y=function(e,t,n){let a=arguments.length>3&&void 0!==arguments[3]?arguments[3]:i.createElement(v.Z,null),r=arguments.length>4&&void 0!==arguments[4]&&arguments[4];if("boolean"==typeof e?!e:void 0===t?!r:!1===t||null===t)return[!1,null];let o="boolean"==typeof t||null==t?a:t;return[!0,n?n(o):o]},K=n(22127),Z=n(86718),X=n(47137),Q=n(92801),J=n(48563);function ee(){}let et=i.createContext({add:ee,remove:ee});var en=n(17094),ea=()=>{let{cancelButtonProps:e,cancelTextLocale:t,onCancel:n}=(0,i.useContext)(R);return i.createElement(y.ZP,Object.assign({onClick:n},e),t)},er=()=>{let{confirmLoading:e,okButtonProps:t,okType:n,okTextLocale:a,onOk:r}=(0,i.useContext)(R);return i.createElement(y.ZP,Object.assign({},(0,T.nx)(n),{loading:e,onClick:r},t),a)},ei=n(4678);function eo(e,t){return i.createElement("span",{className:"".concat(e,"-close-x")},t||i.createElement(v.Z,{className:"".concat(e,"-close-icon")}))}let es=e=>{let t;let{okText:n,okType:a="primary",cancelText:o,confirmLoading:s,onOk:l,onCancel:c,okButtonProps:d,cancelButtonProps:u,footer:p}=e,[g]=(0,E.Z)("Modal",(0,ei.A)()),m={confirmLoading:s,okButtonProps:d,cancelButtonProps:u,okTextLocale:n||(null==g?void 0:g.okText),cancelTextLocale:o||(null==g?void 0:g.cancelText),okType:a,onOk:l,onCancel:c},b=i.useMemo(()=>m,(0,r.Z)(Object.values(m)));return"function"==typeof p||void 0===p?(t=i.createElement(i.Fragment,null,i.createElement(ea,null),i.createElement(er,null)),"function"==typeof p&&(t=p(t,{OkBtn:er,CancelBtn:ea})),t=i.createElement(I,{value:b},t)):t=p,i.createElement(en.n,{disabled:!1},t)};var el=n(11303),ec=n(13703),ed=n(58854),eu=n(80316),ep=n(76585),eg=n(8985);function em(e){return{position:e,inset:0}}let eb=e=>{let{componentCls:t,antCls:n}=e;return[{["".concat(t,"-root")]:{["".concat(t).concat(n,"-zoom-enter, ").concat(t).concat(n,"-zoom-appear")]:{transform:"none",opacity:0,animationDuration:e.motionDurationSlow,userSelect:"none"},["".concat(t).concat(n,"-zoom-leave ").concat(t,"-content")]:{pointerEvents:"none"},["".concat(t,"-mask")]:Object.assign(Object.assign({},em("fixed")),{zIndex:e.zIndexPopupBase,height:"100%",backgroundColor:e.colorBgMask,pointerEvents:"none",["".concat(t,"-hidden")]:{display:"none"}}),["".concat(t,"-wrap")]:Object.assign(Object.assign({},em("fixed")),{zIndex:e.zIndexPopupBase,overflow:"auto",outline:0,WebkitOverflowScrolling:"touch",["&:has(".concat(t).concat(n,"-zoom-enter), &:has(").concat(t).concat(n,"-zoom-appear)")]:{pointerEvents:"none"}})}},{["".concat(t,"-root")]:(0,ec.J$)(e)}]},ef=e=>{let{componentCls:t}=e;return[{["".concat(t,"-root")]:{["".concat(t,"-wrap-rtl")]:{direction:"rtl"},["".concat(t,"-centered")]:{textAlign:"center","&::before":{display:"inline-block",width:0,height:"100%",verticalAlign:"middle",content:'""'},[t]:{top:0,display:"inline-block",paddingBottom:0,textAlign:"start",verticalAlign:"middle"}},["@media (max-width: ".concat(e.screenSMMax,"px)")]:{[t]:{maxWidth:"calc(100vw - 16px)",margin:"".concat((0,eg.bf)(e.marginXS)," auto")},["".concat(t,"-centered")]:{[t]:{flex:1}}}}},{[t]:Object.assign(Object.assign({},(0,el.Wf)(e)),{pointerEvents:"none",position:"relative",top:100,width:"auto",maxWidth:"calc(100vw - ".concat((0,eg.bf)(e.calc(e.margin).mul(2).equal()),")"),margin:"0 auto",paddingBottom:e.paddingLG,["".concat(t,"-title")]:{margin:0,color:e.titleColor,fontWeight:e.fontWeightStrong,fontSize:e.titleFontSize,lineHeight:e.titleLineHeight,wordWrap:"break-word"},["".concat(t,"-content")]:{position:"relative",backgroundColor:e.contentBg,backgroundClip:"padding-box",border:0,borderRadius:e.borderRadiusLG,boxShadow:e.boxShadow,pointerEvents:"auto",padding:e.contentPadding},["".concat(t,"-close")]:Object.assign({position:"absolute",top:e.calc(e.modalHeaderHeight).sub(e.modalCloseBtnSize).div(2).equal(),insetInlineEnd:e.calc(e.modalHeaderHeight).sub(e.modalCloseBtnSize).div(2).equal(),zIndex:e.calc(e.zIndexPopupBase).add(10).equal(),padding:0,color:e.modalCloseIconColor,fontWeight:e.fontWeightStrong,lineHeight:1,textDecoration:"none",background:"transparent",borderRadius:e.borderRadiusSM,width:e.modalCloseBtnSize,height:e.modalCloseBtnSize,border:0,outline:0,cursor:"pointer",transition:"color ".concat(e.motionDurationMid,", background-color ").concat(e.motionDurationMid),"&-x":{display:"flex",fontSize:e.fontSizeLG,fontStyle:"normal",lineHeight:"".concat((0,eg.bf)(e.modalCloseBtnSize)),justifyContent:"center",textTransform:"none",textRendering:"auto"},"&:hover":{color:e.modalIconHoverColor,backgroundColor:e.closeBtnHoverBg,textDecoration:"none"},"&:active":{backgroundColor:e.closeBtnActiveBg}},(0,el.Qy)(e)),["".concat(t,"-header")]:{color:e.colorText,background:e.headerBg,borderRadius:"".concat((0,eg.bf)(e.borderRadiusLG)," ").concat((0,eg.bf)(e.borderRadiusLG)," 0 0"),marginBottom:e.headerMarginBottom,padding:e.headerPadding,borderBottom:e.headerBorderBottom},["".concat(t,"-body")]:{fontSize:e.fontSize,lineHeight:e.lineHeight,wordWrap:"break-word",padding:e.bodyPadding},["".concat(t,"-footer")]:{textAlign:"end",background:e.footerBg,marginTop:e.footerMarginTop,padding:e.footerPadding,borderTop:e.footerBorderTop,borderRadius:e.footerBorderRadius,["> ".concat(e.antCls,"-btn + ").concat(e.antCls,"-btn")]:{marginInlineStart:e.marginXS}},["".concat(t,"-open")]:{overflow:"hidden"}})},{["".concat(t,"-pure-panel")]:{top:"auto",padding:0,display:"flex",flexDirection:"column",["".concat(t,"-content,\n ").concat(t,"-body,\n ").concat(t,"-confirm-body-wrapper")]:{display:"flex",flexDirection:"column",flex:"auto"},["".concat(t,"-confirm-body")]:{marginBottom:"auto"}}}]},eE=e=>{let{componentCls:t}=e;return{["".concat(t,"-root")]:{["".concat(t,"-wrap-rtl")]:{direction:"rtl",["".concat(t,"-confirm-body")]:{direction:"rtl"}}}}},eh=e=>{let t=e.padding,n=e.fontSizeHeading5,a=e.lineHeightHeading5;return(0,eu.TS)(e,{modalHeaderHeight:e.calc(e.calc(a).mul(n).equal()).add(e.calc(t).mul(2).equal()).equal(),modalFooterBorderColorSplit:e.colorSplit,modalFooterBorderStyle:e.lineType,modalFooterBorderWidth:e.lineWidth,modalIconHoverColor:e.colorIconHover,modalCloseIconColor:e.colorIcon,modalCloseBtnSize:e.fontHeight,modalConfirmIconSize:e.fontHeight,modalTitleHeight:e.calc(e.titleFontSize).mul(e.titleLineHeight).equal()})},eS=e=>({footerBg:"transparent",headerBg:e.colorBgElevated,titleLineHeight:e.lineHeightHeading5,titleFontSize:e.fontSizeHeading5,contentBg:e.colorBgElevated,titleColor:e.colorTextHeading,closeBtnHoverBg:e.wireframe?"transparent":e.colorFillContent,closeBtnActiveBg:e.wireframe?"transparent":e.colorFillContentHover,contentPadding:e.wireframe?0:"".concat((0,eg.bf)(e.paddingMD)," ").concat((0,eg.bf)(e.paddingContentHorizontalLG)),headerPadding:e.wireframe?"".concat((0,eg.bf)(e.padding)," ").concat((0,eg.bf)(e.paddingLG)):0,headerBorderBottom:e.wireframe?"".concat((0,eg.bf)(e.lineWidth)," ").concat(e.lineType," ").concat(e.colorSplit):"none",headerMarginBottom:e.wireframe?0:e.marginXS,bodyPadding:e.wireframe?e.paddingLG:0,footerPadding:e.wireframe?"".concat((0,eg.bf)(e.paddingXS)," ").concat((0,eg.bf)(e.padding)):0,footerBorderTop:e.wireframe?"".concat((0,eg.bf)(e.lineWidth)," ").concat(e.lineType," ").concat(e.colorSplit):"none",footerBorderRadius:e.wireframe?"0 0 ".concat((0,eg.bf)(e.borderRadiusLG)," ").concat((0,eg.bf)(e.borderRadiusLG)):0,footerMarginTop:e.wireframe?0:e.marginSM,confirmBodyPadding:e.wireframe?"".concat((0,eg.bf)(2*e.padding)," ").concat((0,eg.bf)(2*e.padding)," ").concat((0,eg.bf)(e.paddingLG)):0,confirmIconMarginInlineEnd:e.wireframe?e.margin:e.marginSM,confirmBtnsMarginTop:e.wireframe?e.marginLG:e.marginSM});var ey=(0,ep.I$)("Modal",e=>{let t=eh(e);return[ef(t),eE(t),eb(t),(0,ed._y)(t,"zoom")]},eS,{unitless:{titleLineHeight:!0}}),eT=n(92935),eA=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n};(0,K.Z)()&&window.document.documentElement&&document.documentElement.addEventListener("click",e=>{a={x:e.pageX,y:e.pageY},setTimeout(()=>{a=null},100)},!0);var eR=e=>{var t;let{getPopupContainer:n,getPrefixCls:r,direction:o,modal:l}=i.useContext(s.E_),c=t=>{let{onCancel:n}=e;null==n||n(t)},{prefixCls:d,className:u,rootClassName:p,open:g,wrapClassName:E,centered:h,getContainer:S,closeIcon:y,closable:T,focusTriggerAfterClose:A=!0,style:R,visible:I,width:N=520,footer:_,classNames:w,styles:k}=e,C=eA(e,["prefixCls","className","rootClassName","open","wrapClassName","centered","getContainer","closeIcon","closable","focusTriggerAfterClose","style","visible","width","footer","classNames","styles"]),O=r("modal",d),x=r(),L=(0,eT.Z)(O),[D,P,M]=ey(O,L),F=m()(E,{["".concat(O,"-centered")]:!!h,["".concat(O,"-wrap-rtl")]:"rtl"===o}),U=null!==_&&i.createElement(es,Object.assign({},e,{onOk:t=>{let{onOk:n}=e;null==n||n(t)},onCancel:c})),[B,G]=Y(T,y,e=>eo(O,e),i.createElement(v.Z,{className:"".concat(O,"-close-icon")}),!0),$=function(e){let t=i.useContext(et),n=i.useRef();return(0,J.zX)(a=>{if(a){let r=e?a.querySelector(e):a;t.add(r),n.current=r}else t.remove(n.current)})}(".".concat(O,"-content")),[H,z]=(0,b.Cn)("Modal",C.zIndex);return D(i.createElement(Q.BR,null,i.createElement(X.Ux,{status:!0,override:!0},i.createElement(Z.Z.Provider,{value:z},i.createElement(q,Object.assign({width:N},C,{zIndex:H,getContainer:void 0===S?n:S,prefixCls:O,rootClassName:m()(P,p,M,L),footer:U,visible:null!=g?g:I,mousePosition:null!==(t=C.mousePosition)&&void 0!==t?t:a,onClose:c,closable:B,closeIcon:G,focusTriggerAfterClose:A,transitionName:(0,f.m)(x,"zoom",e.transitionName),maskTransitionName:(0,f.m)(x,"fade",e.maskTransitionName),className:m()(P,u,null==l?void 0:l.className),style:Object.assign(Object.assign({},null==l?void 0:l.style),R),classNames:Object.assign(Object.assign({wrapper:F},null==l?void 0:l.classNames),w),styles:Object.assign(Object.assign({},null==l?void 0:l.styles),k),panelRef:$}))))))};let eI=e=>{let{componentCls:t,titleFontSize:n,titleLineHeight:a,modalConfirmIconSize:r,fontSize:i,lineHeight:o,modalTitleHeight:s,fontHeight:l,confirmBodyPadding:c}=e,d="".concat(t,"-confirm");return{[d]:{"&-rtl":{direction:"rtl"},["".concat(e.antCls,"-modal-header")]:{display:"none"},["".concat(d,"-body-wrapper")]:Object.assign({},(0,el.dF)()),["&".concat(t," ").concat(t,"-body")]:{padding:c},["".concat(d,"-body")]:{display:"flex",flexWrap:"nowrap",alignItems:"start",["> ".concat(e.iconCls)]:{flex:"none",fontSize:r,marginInlineEnd:e.confirmIconMarginInlineEnd,marginTop:e.calc(e.calc(l).sub(r).equal()).div(2).equal()},["&-has-title > ".concat(e.iconCls)]:{marginTop:e.calc(e.calc(s).sub(r).equal()).div(2).equal()}},["".concat(d,"-paragraph")]:{display:"flex",flexDirection:"column",flex:"auto",rowGap:e.marginXS,maxWidth:"calc(100% - ".concat((0,eg.bf)(e.calc(e.modalConfirmIconSize).add(e.marginSM).equal()),")")},["".concat(d,"-title")]:{color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:n,lineHeight:a},["".concat(d,"-content")]:{color:e.colorText,fontSize:i,lineHeight:o},["".concat(d,"-btns")]:{textAlign:"end",marginTop:e.confirmBtnsMarginTop,["".concat(e.antCls,"-btn + ").concat(e.antCls,"-btn")]:{marginBottom:0,marginInlineStart:e.marginXS}}},["".concat(d,"-error ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorError},["".concat(d,"-warning ").concat(d,"-body > ").concat(e.iconCls,",\n ").concat(d,"-confirm ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorWarning},["".concat(d,"-info ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorInfo},["".concat(d,"-success ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorSuccess}}};var eN=(0,ep.bk)(["Modal","confirm"],e=>[eI(eh(e))],eS,{order:-1e3}),e_=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n};function ev(e){let{prefixCls:t,icon:n,okText:a,cancelText:o,confirmPrefixCls:s,type:l,okCancel:g,footer:b,locale:f}=e,h=e_(e,["prefixCls","icon","okText","cancelText","confirmPrefixCls","type","okCancel","footer","locale"]),S=n;if(!n&&null!==n)switch(l){case"info":S=i.createElement(p.Z,null);break;case"success":S=i.createElement(c.Z,null);break;case"error":S=i.createElement(d.Z,null);break;default:S=i.createElement(u.Z,null)}let y=null!=g?g:"confirm"===l,T=null!==e.autoFocusButton&&(e.autoFocusButton||"ok"),[A]=(0,E.Z)("Modal"),R=f||A,v=a||(y?null==R?void 0:R.okText:null==R?void 0:R.justOkText),w=Object.assign({autoFocusButton:T,cancelTextLocale:o||(null==R?void 0:R.cancelText),okTextLocale:v,mergedOkCancel:y},h),k=i.useMemo(()=>w,(0,r.Z)(Object.values(w))),C=i.createElement(i.Fragment,null,i.createElement(N,null),i.createElement(_,null)),O=void 0!==e.title&&null!==e.title,x="".concat(s,"-body");return i.createElement("div",{className:"".concat(s,"-body-wrapper")},i.createElement("div",{className:m()(x,{["".concat(x,"-has-title")]:O})},S,i.createElement("div",{className:"".concat(s,"-paragraph")},O&&i.createElement("span",{className:"".concat(s,"-title")},e.title),i.createElement("div",{className:"".concat(s,"-content")},e.content))),void 0===b||"function"==typeof b?i.createElement(I,{value:k},i.createElement("div",{className:"".concat(s,"-btns")},"function"==typeof b?b(C,{OkBtn:_,CancelBtn:N}):C)):b,i.createElement(eN,{prefixCls:t}))}let ew=e=>{let{close:t,zIndex:n,afterClose:a,open:r,keyboard:o,centered:s,getContainer:l,maskStyle:c,direction:d,prefixCls:u,wrapClassName:p,rootPrefixCls:g,bodyStyle:E,closable:S=!1,closeIcon:y,modalRender:T,focusTriggerAfterClose:A,onConfirm:R,styles:I}=e,N="".concat(u,"-confirm"),_=e.width||416,v=e.style||{},w=void 0===e.mask||e.mask,k=void 0!==e.maskClosable&&e.maskClosable,C=m()(N,"".concat(N,"-").concat(e.type),{["".concat(N,"-rtl")]:"rtl"===d},e.className),[,O]=(0,h.ZP)(),x=i.useMemo(()=>void 0!==n?n:O.zIndexPopupBase+b.u6,[n,O]);return i.createElement(eR,{prefixCls:u,className:C,wrapClassName:m()({["".concat(N,"-centered")]:!!e.centered},p),onCancel:()=>{null==t||t({triggerCancel:!0}),null==R||R(!1)},open:r,title:"",footer:null,transitionName:(0,f.m)(g||"","zoom",e.transitionName),maskTransitionName:(0,f.m)(g||"","fade",e.maskTransitionName),mask:w,maskClosable:k,style:v,styles:Object.assign({body:E,mask:c},I),width:_,zIndex:x,afterClose:a,keyboard:o,centered:s,getContainer:l,closable:S,closeIcon:y,modalRender:T,focusTriggerAfterClose:A},i.createElement(ev,Object.assign({},e,{confirmPrefixCls:N})))};var ek=e=>{let{rootPrefixCls:t,iconPrefixCls:n,direction:a,theme:r}=e;return i.createElement(l.ZP,{prefixCls:t,iconPrefixCls:n,direction:a,theme:r},i.createElement(ew,Object.assign({},e)))},eC=[];let eO="",ex=e=>{var t,n;let{prefixCls:a,getContainer:r,direction:o}=e,l=(0,ei.A)(),c=(0,i.useContext)(s.E_),d=eO||c.getPrefixCls(),u=a||"".concat(d,"-modal"),p=r;return!1===p&&(p=void 0),i.createElement(ek,Object.assign({},e,{rootPrefixCls:d,prefixCls:u,iconPrefixCls:c.iconPrefixCls,theme:c.theme,direction:null!=o?o:c.direction,locale:null!==(n=null===(t=c.locale)||void 0===t?void 0:t.Modal)&&void 0!==n?n:l,getContainer:p}))};function eL(e){let t;let n=(0,l.w6)(),a=document.createDocumentFragment(),s=Object.assign(Object.assign({},e),{close:u,open:!0});function c(){for(var t=arguments.length,n=Array(t),i=0;ie&&e.triggerCancel);e.onCancel&&s&&e.onCancel.apply(e,[()=>{}].concat((0,r.Z)(n.slice(1))));for(let e=0;e{let t=n.getPrefixCls(void 0,eO),r=n.getIconPrefixCls(),s=n.getTheme(),c=i.createElement(ex,Object.assign({},e));(0,o.s)(i.createElement(l.ZP,{prefixCls:t,iconPrefixCls:r,theme:s},n.holderRender?n.holderRender(c):c),a)})}function u(){for(var t=arguments.length,n=Array(t),a=0;a{"function"==typeof e.afterClose&&e.afterClose(),c.apply(this,n)}})).visible&&delete s.visible,d(s)}return d(s),eC.push(u),{destroy:u,update:function(e){d(s="function"==typeof e?e(s):Object.assign(Object.assign({},s),e))}}}function eD(e){return Object.assign(Object.assign({},e),{type:"warning"})}function eP(e){return Object.assign(Object.assign({},e),{type:"info"})}function eM(e){return Object.assign(Object.assign({},e),{type:"success"})}function eF(e){return Object.assign(Object.assign({},e),{type:"error"})}function eU(e){return Object.assign(Object.assign({},e),{type:"confirm"})}var eB=n(21467),eG=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n},e$=(0,eB.i)(e=>{let{prefixCls:t,className:n,closeIcon:a,closable:r,type:o,title:l,children:c,footer:d}=e,u=eG(e,["prefixCls","className","closeIcon","closable","type","title","children","footer"]),{getPrefixCls:p}=i.useContext(s.E_),g=p(),b=t||p("modal"),f=(0,eT.Z)(g),[E,h,S]=ey(b,f),y="".concat(b,"-confirm"),T={};return T=o?{closable:null!=r&&r,title:"",footer:"",children:i.createElement(ev,Object.assign({},e,{prefixCls:b,confirmPrefixCls:y,rootPrefixCls:g,content:c}))}:{closable:null==r||r,title:l,footer:null!==d&&i.createElement(es,Object.assign({},e)),children:c},E(i.createElement(z,Object.assign({prefixCls:b,className:m()(h,"".concat(b,"-pure-panel"),o&&y,o&&"".concat(y,"-").concat(o),n,S,f)},u,{closeIcon:eo(b,a),closable:r},T)))}),eH=n(79474),ez=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n},ej=i.forwardRef((e,t)=>{var n,{afterClose:a,config:o}=e,l=ez(e,["afterClose","config"]);let[c,d]=i.useState(!0),[u,p]=i.useState(o),{direction:g,getPrefixCls:m}=i.useContext(s.E_),b=m("modal"),f=m(),h=function(){d(!1);for(var e=arguments.length,t=Array(e),n=0;ne&&e.triggerCancel);u.onCancel&&a&&u.onCancel.apply(u,[()=>{}].concat((0,r.Z)(t.slice(1))))};i.useImperativeHandle(t,()=>({destroy:h,update:e=>{p(t=>Object.assign(Object.assign({},t),e))}}));let S=null!==(n=u.okCancel)&&void 0!==n?n:"confirm"===u.type,[y]=(0,E.Z)("Modal",eH.Z.Modal);return i.createElement(ek,Object.assign({prefixCls:b,rootPrefixCls:f},u,{close:h,open:c,afterClose:()=>{var e;a(),null===(e=u.afterClose)||void 0===e||e.call(u)},okText:u.okText||(S?null==y?void 0:y.okText:null==y?void 0:y.justOkText),direction:u.direction||g,cancelText:u.cancelText||(null==y?void 0:y.cancelText)},l))});let eV=0,eW=i.memo(i.forwardRef((e,t)=>{let[n,a]=function(){let[e,t]=i.useState([]);return[e,i.useCallback(e=>(t(t=>[].concat((0,r.Z)(t),[e])),()=>{t(t=>t.filter(t=>t!==e))}),[])]}();return i.useImperativeHandle(t,()=>({patchElement:a}),[]),i.createElement(i.Fragment,null,n)}));function eq(e){return eL(eD(e))}eR.useModal=function(){let e=i.useRef(null),[t,n]=i.useState([]);i.useEffect(()=>{t.length&&((0,r.Z)(t).forEach(e=>{e()}),n([]))},[t]);let a=i.useCallback(t=>function(a){var o;let s,l;eV+=1;let c=i.createRef(),d=new Promise(e=>{s=e}),u=!1,p=i.createElement(ej,{key:"modal-".concat(eV),config:t(a),ref:c,afterClose:()=>{null==l||l()},isSilent:()=>u,onConfirm:e=>{s(e)}});return(l=null===(o=e.current)||void 0===o?void 0:o.patchElement(p))&&eC.push(l),{destroy:()=>{function e(){var e;null===(e=c.current)||void 0===e||e.destroy()}c.current?e():n(t=>[].concat((0,r.Z)(t),[e]))},update:e=>{function t(){var t;null===(t=c.current)||void 0===t||t.update(e)}c.current?t():n(e=>[].concat((0,r.Z)(e),[t]))},then:e=>(u=!0,d.then(e))}},[]);return[i.useMemo(()=>({info:a(eP),success:a(eM),error:a(eF),warning:a(eD),confirm:a(eU)}),[]),i.createElement(eW,{key:"modal-holder",ref:e})]},eR.info=function(e){return eL(eP(e))},eR.success=function(e){return eL(eM(e))},eR.error=function(e){return eL(eF(e))},eR.warning=eq,eR.warn=eq,eR.confirm=function(e){return eL(eU(e))},eR.destroyAll=function(){for(;eC.length;){let e=eC.pop();e&&e()}},eR.config=function(e){let{rootPrefixCls:t}=e;eO=t},eR._InternalPanelDoNotUseOrYouWillBeFired=e$;var eY=eR},13703:function(e,t,n){n.d(t,{J$:function(){return s}});var a=n(8985),r=n(59353);let i=new a.E4("antFadeIn",{"0%":{opacity:0},"100%":{opacity:1}}),o=new a.E4("antFadeOut",{"0%":{opacity:1},"100%":{opacity:0}}),s=function(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],{antCls:n}=e,a="".concat(n,"-fade"),s=t?"&":"";return[(0,r.R)(a,i,o,e.motionDurationMid,t),{["\n ".concat(s).concat(a,"-enter,\n ").concat(s).concat(a,"-appear\n ")]:{opacity:0,animationTimingFunction:"linear"},["".concat(s).concat(a,"-leave")]:{animationTimingFunction:"linear"}}]}},44056:function(e){e.exports=function(e,n){for(var a,r,i,o=e||"",s=n||"div",l={},c=0;c4&&m.slice(0,4)===o&&s.test(t)&&("-"===t.charAt(4)?b=o+(n=t.slice(5).replace(l,u)).charAt(0).toUpperCase()+n.slice(1):(g=(p=t).slice(4),t=l.test(g)?p:("-"!==(g=g.replace(c,d)).charAt(0)&&(g="-"+g),o+g)),f=r),new f(b,t))};var s=/^data[-\w.:]+$/i,l=/-[a-z]/g,c=/[A-Z]/g;function d(e){return"-"+e.toLowerCase()}function u(e){return e.charAt(1).toUpperCase()}},31872:function(e,t,n){var a=n(96130),r=n(64730),i=n(61861),o=n(46982),s=n(83671),l=n(53618);e.exports=a([i,r,o,s,l])},83671:function(e,t,n){var a=n(7667),r=n(13585),i=a.booleanish,o=a.number,s=a.spaceSeparated;e.exports=r({transform:function(e,t){return"role"===t?t:"aria-"+t.slice(4).toLowerCase()},properties:{ariaActiveDescendant:null,ariaAtomic:i,ariaAutoComplete:null,ariaBusy:i,ariaChecked:i,ariaColCount:o,ariaColIndex:o,ariaColSpan:o,ariaControls:s,ariaCurrent:null,ariaDescribedBy:s,ariaDetails:null,ariaDisabled:i,ariaDropEffect:s,ariaErrorMessage:null,ariaExpanded:i,ariaFlowTo:s,ariaGrabbed:i,ariaHasPopup:null,ariaHidden:i,ariaInvalid:null,ariaKeyShortcuts:null,ariaLabel:null,ariaLabelledBy:s,ariaLevel:o,ariaLive:null,ariaModal:i,ariaMultiLine:i,ariaMultiSelectable:i,ariaOrientation:null,ariaOwns:s,ariaPlaceholder:null,ariaPosInSet:o,ariaPressed:i,ariaReadOnly:i,ariaRelevant:null,ariaRequired:i,ariaRoleDescription:s,ariaRowCount:o,ariaRowIndex:o,ariaRowSpan:o,ariaSelected:i,ariaSetSize:o,ariaSort:null,ariaValueMax:o,ariaValueMin:o,ariaValueNow:o,ariaValueText:null,role:null}})},53618:function(e,t,n){var a=n(7667),r=n(13585),i=n(46640),o=a.boolean,s=a.overloadedBoolean,l=a.booleanish,c=a.number,d=a.spaceSeparated,u=a.commaSeparated;e.exports=r({space:"html",attributes:{acceptcharset:"accept-charset",classname:"class",htmlfor:"for",httpequiv:"http-equiv"},transform:i,mustUseProperty:["checked","multiple","muted","selected"],properties:{abbr:null,accept:u,acceptCharset:d,accessKey:d,action:null,allow:null,allowFullScreen:o,allowPaymentRequest:o,allowUserMedia:o,alt:null,as:null,async:o,autoCapitalize:null,autoComplete:d,autoFocus:o,autoPlay:o,capture:o,charSet:null,checked:o,cite:null,className:d,cols:c,colSpan:null,content:null,contentEditable:l,controls:o,controlsList:d,coords:c|u,crossOrigin:null,data:null,dateTime:null,decoding:null,default:o,defer:o,dir:null,dirName:null,disabled:o,download:s,draggable:l,encType:null,enterKeyHint:null,form:null,formAction:null,formEncType:null,formMethod:null,formNoValidate:o,formTarget:null,headers:d,height:c,hidden:o,high:c,href:null,hrefLang:null,htmlFor:d,httpEquiv:d,id:null,imageSizes:null,imageSrcSet:u,inputMode:null,integrity:null,is:null,isMap:o,itemId:null,itemProp:d,itemRef:d,itemScope:o,itemType:d,kind:null,label:null,lang:null,language:null,list:null,loading:null,loop:o,low:c,manifest:null,max:null,maxLength:c,media:null,method:null,min:null,minLength:c,multiple:o,muted:o,name:null,nonce:null,noModule:o,noValidate:o,onAbort:null,onAfterPrint:null,onAuxClick:null,onBeforePrint:null,onBeforeUnload:null,onBlur:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onContextMenu:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnded:null,onError:null,onFocus:null,onFormData:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLanguageChange:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadEnd:null,onLoadStart:null,onMessage:null,onMessageError:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRejectionHandled:null,onReset:null,onResize:null,onScroll:null,onSecurityPolicyViolation:null,onSeeked:null,onSeeking:null,onSelect:null,onSlotChange:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnhandledRejection:null,onUnload:null,onVolumeChange:null,onWaiting:null,onWheel:null,open:o,optimum:c,pattern:null,ping:d,placeholder:null,playsInline:o,poster:null,preload:null,readOnly:o,referrerPolicy:null,rel:d,required:o,reversed:o,rows:c,rowSpan:c,sandbox:d,scope:null,scoped:o,seamless:o,selected:o,shape:null,size:c,sizes:null,slot:null,span:c,spellCheck:l,src:null,srcDoc:null,srcLang:null,srcSet:u,start:c,step:null,style:null,tabIndex:c,target:null,title:null,translate:null,type:null,typeMustMatch:o,useMap:null,value:l,width:c,wrap:null,align:null,aLink:null,archive:d,axis:null,background:null,bgColor:null,border:c,borderColor:null,bottomMargin:c,cellPadding:null,cellSpacing:null,char:null,charOff:null,classId:null,clear:null,code:null,codeBase:null,codeType:null,color:null,compact:o,declare:o,event:null,face:null,frame:null,frameBorder:null,hSpace:c,leftMargin:c,link:null,longDesc:null,lowSrc:null,marginHeight:c,marginWidth:c,noResize:o,noHref:o,noShade:o,noWrap:o,object:null,profile:null,prompt:null,rev:null,rightMargin:c,rules:null,scheme:null,scrolling:l,standby:null,summary:null,text:null,topMargin:c,valueType:null,version:null,vAlign:null,vLink:null,vSpace:c,allowTransparency:null,autoCorrect:null,autoSave:null,disablePictureInPicture:o,disableRemotePlayback:o,prefix:null,property:null,results:c,security:null,unselectable:null}})},46640:function(e,t,n){var a=n(25852);e.exports=function(e,t){return a(e,t.toLowerCase())}},25852:function(e){e.exports=function(e,t){return t in e?e[t]:t}},13585:function(e,t,n){var a=n(39900),r=n(94949),i=n(7478);e.exports=function(e){var t,n,o=e.space,s=e.mustUseProperty||[],l=e.attributes||{},c=e.properties,d=e.transform,u={},p={};for(t in c)n=new i(t,d(l,t),c[t],o),-1!==s.indexOf(t)&&(n.mustUseProperty=!0),u[t]=n,p[a(t)]=t,p[a(n.attribute)]=t;return new r(u,p,o)}},7478:function(e,t,n){var a=n(74108),r=n(7667);e.exports=s,s.prototype=new a,s.prototype.defined=!0;var i=["boolean","booleanish","overloadedBoolean","number","commaSeparated","spaceSeparated","commaOrSpaceSeparated"],o=i.length;function s(e,t,n,s){var l,c,d,u=-1;for(s&&(this.space=s),a.call(this,e,t);++u