From 71739e942ac6ba61bf86d3987612d46c9d603181 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 09:25:57 -0700 Subject: [PATCH 01/48] fix qdrant semantic cache --- docs/my-website/docs/caching/all_caches.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index 334adea3a..8b4d7e863 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -193,7 +193,7 @@ response2 = completion( ], max_tokens=20, ) -print(f"response2: {response1}") +print(f"response2: {response2}") assert response1.id == response2.id # response1 == response2, response 1 is cached ``` From 68bb735b3b0d15539821521ffb5a0da6fcae069d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 10:50:24 -0700 Subject: [PATCH 02/48] docs use litellm proxy with litellm python sdk --- .../docs/providers/litellm_proxy.md | 90 +++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 91 insertions(+) create mode 100644 docs/my-website/docs/providers/litellm_proxy.md diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md new file mode 100644 index 000000000..de026967f --- /dev/null +++ b/docs/my-website/docs/providers/litellm_proxy.md @@ -0,0 +1,90 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# LiteLLM Proxy (LLM Gateway) + + +**[LiteLLM Proxy](../simple_proxy) is OpenAI compatible**, you just need the `openai/` prefix before the model + +:::tip + +[LiteLLM Providers a **self hosted** proxy server (AI Gateway)](../simple_proxy) to call all the LLMs in the OpenAI format + +::: + +## Required Variables + +```python +os.environ["OPENAI_API_KEY"] = "" # "sk-1234" your litellm proxy api key +os.environ["OPENAI_API_BASE"] = "" # "http://localhost:4000" your litellm proxy api base +``` + + +## Usage (Non Streaming) +```python +import os +import litellm +from litellm import completion + +os.environ["OPENAI_API_KEY"] = "" + +# set custom api base to your proxy +# either set .env or litellm.api_base +# os.environ["OPENAI_API_BASE"] = "" +litellm.api_base = "your-openai-proxy-url" + + +messages = [{ "content": "Hello, how are you?","role": "user"}] + +# openai call +response = completion(model="openai/your-model-name", messages) +``` + +## Usage - passing `api_base`, `api_key` per request + +If you need to set api_base dynamically, just pass it in completions instead - completions(...,api_base="your-proxy-api-base") + +```python +import os +import litellm +from litellm import completion + +os.environ["OPENAI_API_KEY"] = "" + +messages = [{ "content": "Hello, how are you?","role": "user"}] + +# openai call +response = completion( + model="openai/your-model-name", + messages, + api_base = "your-litellm-proxy-url", + api_key = "your-litellm-proxy-api-key" +) +``` +## Usage - Streaming + +```python +import os +import litellm +from litellm import completion + +os.environ["OPENAI_API_KEY"] = "" + +messages = [{ "content": "Hello, how are you?","role": "user"}] + +# openai call +response = completion( + model="openai/your-model-name", + messages, + api_base = "your-litellm-proxy-url", + stream=True +) + +for chunk in response: + print(chunk) +``` + + +## **Usage with Langchain, LLamaindex, OpenAI Js, Anthropic SDK, Instructor** + +[Follow this doc to see how to use litellm proxy with langchain, llamaindex, anthropic etc](../proxy/user_keys) \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8c8c87fb8..beae3544f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -128,6 +128,7 @@ const sidebars = { "providers/anthropic", "providers/aws_sagemaker", "providers/bedrock", + "providers/litellm_proxy", "providers/mistral", "providers/codestral", "providers/cohere", From d13d2e8a623baa9a9e811003075abfc1173809dc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 10:59:01 -0700 Subject: [PATCH 03/48] feat(vertex_httpx.py): support functions param for gemini google ai studio + vertex ai Closes https://github.com/BerriAI/litellm/issues/5344 --- litellm/llms/vertex_httpx.py | 180 +++++++++++++++++++++---------- litellm/tests/test_completion.py | 94 +++++++++++++--- litellm/types/llms/openai.py | 1 + litellm/types/llms/vertex_ai.py | 2 +- 4 files changed, 204 insertions(+), 73 deletions(-) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index faecdd790..1e842ce57 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -30,6 +30,7 @@ from litellm.types.llms.openai import ( ChatCompletionResponseMessage, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, + ChatCompletionToolParamFunctionChunk, ChatCompletionUsageBlock, ) from litellm.types.llms.vertex_ai import ( @@ -296,11 +297,50 @@ class GoogleAIStudioGeminiConfig: # key diff from VertexAI - 'frequency_penalty "stream", "tools", "tool_choice", + "functions", "response_format", "n", "stop", ] + def _map_function(self, value: List[dict]) -> List[Tools]: + gtool_func_declarations = [] + googleSearchRetrieval: Optional[dict] = None + + for tool in value: + openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = ( + None + ) + if "function" in tool: # tools list + openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore + **tool["function"] + ) + elif "name" in tool: # functions list + openai_function_object = ChatCompletionToolParamFunctionChunk(**tool) # type: ignore + + # check if grounding + if tool.get("googleSearchRetrieval", None) is not None: + googleSearchRetrieval = tool["googleSearchRetrieval"] + elif openai_function_object is not None: + gtool_func_declaration = FunctionDeclaration( + name=openai_function_object["name"], + description=openai_function_object.get("description", ""), + parameters=openai_function_object.get("parameters", {}), + ) + gtool_func_declarations.append(gtool_func_declaration) + else: + # assume it's a provider-specific param + verbose_logger.warning( + "Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request." + ) + + _tools = Tools( + function_declarations=gtool_func_declarations, + ) + if googleSearchRetrieval is not None: + _tools["googleSearchRetrieval"] = googleSearchRetrieval + return [_tools] + def map_tool_choice_values( self, model: str, tool_choice: Union[str, dict] ) -> Optional[ToolConfig]: @@ -363,26 +403,11 @@ class GoogleAIStudioGeminiConfig: # key diff from VertexAI - 'frequency_penalty if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore optional_params["response_mime_type"] = "application/json" optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore - if param == "tools" and isinstance(value, list): - gtool_func_declarations = [] - for tool in value: - _parameters = tool.get("function", {}).get("parameters", {}) - _properties = _parameters.get("properties", {}) - if isinstance(_properties, dict): - for _, _property in _properties.items(): - if "enum" in _property and "format" not in _property: - _property["format"] = "enum" - - gtool_func_declaration = FunctionDeclaration( - name=tool["function"]["name"], - description=tool["function"].get("description", ""), - ) - if len(_parameters.keys()) > 0: - gtool_func_declaration["parameters"] = _parameters - gtool_func_declarations.append(gtool_func_declaration) - optional_params["tools"] = [ - Tools(function_declarations=gtool_func_declarations) - ] + if (param == "tools" or param == "functions") and isinstance(value, list): + optional_params["tools"] = self._map_function(value=value) + optional_params["litellm_param_is_function_call"] = ( + True if param == "functions" else False + ) if param == "tool_choice" and ( isinstance(value, str) or isinstance(value, dict) ): @@ -506,6 +531,7 @@ class VertexGeminiConfig: "max_tokens", "stream", "tools", + "functions", "tool_choice", "response_format", "n", @@ -541,6 +567,44 @@ class VertexGeminiConfig: status_code=400, ) + def _map_function(self, value: List[dict]) -> List[Tools]: + gtool_func_declarations = [] + googleSearchRetrieval: Optional[dict] = None + + for tool in value: + openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = ( + None + ) + if "function" in tool: # tools list + openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore + **tool["function"] + ) + elif "name" in tool: # functions list + openai_function_object = ChatCompletionToolParamFunctionChunk(**tool) # type: ignore + + # check if grounding + if tool.get("googleSearchRetrieval", None) is not None: + googleSearchRetrieval = tool["googleSearchRetrieval"] + elif openai_function_object is not None: + gtool_func_declaration = FunctionDeclaration( + name=openai_function_object["name"], + description=openai_function_object.get("description", ""), + parameters=openai_function_object.get("parameters", {}), + ) + gtool_func_declarations.append(gtool_func_declaration) + else: + # assume it's a provider-specific param + verbose_logger.warning( + "Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request." + ) + + _tools = Tools( + function_declarations=gtool_func_declarations, + ) + if googleSearchRetrieval is not None: + _tools["googleSearchRetrieval"] = googleSearchRetrieval + return [_tools] + def map_openai_params( self, model: str, @@ -582,33 +646,11 @@ class VertexGeminiConfig: optional_params["frequency_penalty"] = value if param == "presence_penalty": optional_params["presence_penalty"] = value - if param == "tools" and isinstance(value, list): - gtool_func_declarations = [] - googleSearchRetrieval: Optional[dict] = None - provider_specific_tools: List[dict] = [] - for tool in value: - # check if grounding - try: - gtool_func_declaration = FunctionDeclaration( - name=tool["function"]["name"], - description=tool["function"].get("description", ""), - parameters=tool["function"].get("parameters", {}), - ) - gtool_func_declarations.append(gtool_func_declaration) - except KeyError: - if tool.get("googleSearchRetrieval", None) is not None: - googleSearchRetrieval = tool["googleSearchRetrieval"] - else: - # assume it's a provider-specific param - verbose_logger.warning( - "Got KeyError parsing tool={}. Assuming it's a provider-specific param. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request." - ) - _tools = Tools( - function_declarations=gtool_func_declarations, + if (param == "tools" or param == "functions") and isinstance(value, list): + optional_params["tools"] = self._map_function(value=value) + optional_params["is_function_call"] = ( + True if param == "functions" else False ) - if googleSearchRetrieval is not None: - _tools["googleSearchRetrieval"] = googleSearchRetrieval - optional_params["tools"] = [_tools] + provider_specific_tools if param == "tool_choice" and ( isinstance(value, str) or isinstance(value, dict) ): @@ -780,6 +822,7 @@ class VertexLLM(BaseLLM): model_response: ModelResponse, logging_obj: litellm.litellm_core_utils.litellm_logging.Logging, optional_params: dict, + litellm_params: dict, api_key: str, data: Union[dict, str], messages: List, @@ -796,7 +839,6 @@ class VertexLLM(BaseLLM): ) print_verbose(f"raw model_response: {response.text}") - ## RESPONSE OBJECT try: completion_response = GenerateContentResponseBody(**response.json()) # type: ignore @@ -904,6 +946,7 @@ class VertexLLM(BaseLLM): chat_completion_message = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] + functions: Optional[ChatCompletionToolCallFunctionChunk] = None for idx, candidate in enumerate(completion_response["candidates"]): if "content" not in candidate: continue @@ -926,18 +969,25 @@ class VertexLLM(BaseLLM): candidate["content"]["parts"][0]["functionCall"]["args"] ), ) - _tool_response_chunk = ChatCompletionToolCallChunk( - id=f"call_{str(uuid.uuid4())}", - type="function", - function=_function_chunk, - index=candidate.get("index", idx), - ) - tools.append(_tool_response_chunk) + if litellm_params.get("litellm_param_is_function_call") is True: + functions = _function_chunk + else: + _tool_response_chunk = ChatCompletionToolCallChunk( + id=f"call_{str(uuid.uuid4())}", + type="function", + function=_function_chunk, + index=candidate.get("index", idx), + ) + tools.append(_tool_response_chunk) chat_completion_message["content"] = ( content_str if len(content_str) > 0 else None ) - chat_completion_message["tool_calls"] = tools + if len(tools) > 0: + chat_completion_message["tool_calls"] = tools + + if functions is not None: + chat_completion_message["function_call"] = functions choice = litellm.Choices( finish_reason=candidate.get("finishReason", "stop"), @@ -1226,7 +1276,7 @@ class VertexLLM(BaseLLM): logging_obj, stream, optional_params: dict, - litellm_params=None, + litellm_params: dict, logger_fn=None, headers={}, client: Optional[AsyncHTTPHandler] = None, @@ -1260,6 +1310,7 @@ class VertexLLM(BaseLLM): messages=messages, print_verbose=print_verbose, optional_params=optional_params, + litellm_params=litellm_params, encoding=encoding, ) @@ -1281,7 +1332,7 @@ class VertexLLM(BaseLLM): vertex_location: Optional[str], vertex_credentials: Optional[str], gemini_api_key: Optional[str], - litellm_params=None, + litellm_params: dict, logger_fn=None, extra_headers: Optional[dict] = None, client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None, @@ -1293,7 +1344,6 @@ class VertexLLM(BaseLLM): optional_params=optional_params ) - print_verbose("Incoming Vertex Args - {}".format(locals())) auth_header, url = self._get_token_and_url( model=model, gemini_api_key=gemini_api_key, @@ -1305,7 +1355,6 @@ class VertexLLM(BaseLLM): api_base=api_base, should_use_v1beta1_features=should_use_v1beta1_features, ) - print_verbose("Updated URL - {}".format(url)) ## TRANSFORMATION ## try: @@ -1349,6 +1398,18 @@ class VertexLLM(BaseLLM): ) optional_params.pop("response_schema") + # Check for any 'litellm_param_*' set during optional param mapping + + remove_keys = [] + for k, v in optional_params.items(): + if k.startswith("litellm_param_"): + litellm_params.update({k: v}) + remove_keys.append(k) + + optional_params = { + k: v for k, v in optional_params.items() if k not in remove_keys + } + try: content = _gemini_convert_messages_with_history(messages=messages) tools: Optional[Tools] = optional_params.pop("tools", None) @@ -1482,6 +1543,7 @@ class VertexLLM(BaseLLM): model_response=model_response, logging_obj=logging_obj, optional_params=optional_params, + litellm_params=litellm_params, api_key="", data=data, # type: ignore messages=messages, diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 0e96f056b..d5179da7a 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2691,8 +2691,61 @@ def test_completion_hf_model_no_provider(): # test_completion_hf_model_no_provider() -@pytest.mark.skip(reason="anyscale stopped serving public api endpoints") -def test_completion_anyscale_with_functions(): +def gemini_mock_post(*args, **kwargs): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.json = MagicMock( + return_value={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "Boston, MA"}, + } + } + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "safetyRatings": [ + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE", + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "NEGLIGIBLE", + }, + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE", + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "probability": "NEGLIGIBLE", + }, + ], + } + ], + "usageMetadata": { + "promptTokenCount": 86, + "candidatesTokenCount": 19, + "totalTokenCount": 105, + }, + } + ) + + return mock_response + + +@pytest.mark.asyncio +async def test_completion_functions_param(): + litellm.set_verbose = True function1 = [ { "name": "get_current_weather", @@ -2711,18 +2764,33 @@ def test_completion_anyscale_with_functions(): } ] try: - messages = [{"role": "user", "content": "What is the weather like in Boston?"}] - response = completion( - model="anyscale/mistralai/Mistral-7B-Instruct-v0.1", - messages=messages, - functions=function1, - ) - # Add any assertions here to check the response - print(response) + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - cost = litellm.completion_cost(completion_response=response) - print("cost to make anyscale completion=", cost) - assert cost > 0.0 + messages = [{"role": "user", "content": "What is the weather like in Boston?"}] + + client = AsyncHTTPHandler(concurrent_limit=1) + + with patch.object(client, "post", side_effect=gemini_mock_post) as mock_client: + response: litellm.ModelResponse = await litellm.acompletion( + model="gemini/gemini-1.5-pro", + messages=messages, + functions=function1, + client=client, + ) + print(response) + # Add any assertions here to check the response + mock_client.assert_called() + print(f"mock_client.call_args.kwargs: {mock_client.call_args.kwargs}") + assert "tools" in mock_client.call_args.kwargs["json"] + assert ( + "litellm_param_is_function_call" + not in mock_client.call_args.kwargs["json"] + ) + assert ( + "litellm_param_is_function_call" + not in mock_client.call_args.kwargs["json"]["generationConfig"] + ) + assert response.choices[0].message.function_call is not None except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 5d2c416f9..98ce3ae78 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -448,6 +448,7 @@ class ChatCompletionResponseMessage(TypedDict, total=False): content: Optional[str] tool_calls: List[ChatCompletionToolCallChunk] role: Literal["assistant"] + function_call: ChatCompletionToolCallFunctionChunk class ChatCompletionUsageBlock(TypedDict): diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 5586d4861..74acd4fec 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -90,7 +90,7 @@ class Schema(TypedDict, total=False): class FunctionDeclaration(TypedDict, total=False): name: Required[str] description: str - parameters: Schema + parameters: Union[Schema, dict] response: Schema From 6d61c396b3cba8accbee84230066adb151e3d08c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 11:03:33 -0700 Subject: [PATCH 04/48] docs using litellm sdk with litellm proxy --- docs/my-website/docs/providers/litellm_proxy.md | 7 +++---- docs/my-website/docs/proxy/user_keys.md | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md index de026967f..2914b09f2 100644 --- a/docs/my-website/docs/providers/litellm_proxy.md +++ b/docs/my-website/docs/providers/litellm_proxy.md @@ -3,15 +3,14 @@ import TabItem from '@theme/TabItem'; # LiteLLM Proxy (LLM Gateway) - -**[LiteLLM Proxy](../simple_proxy) is OpenAI compatible**, you just need the `openai/` prefix before the model - :::tip [LiteLLM Providers a **self hosted** proxy server (AI Gateway)](../simple_proxy) to call all the LLMs in the OpenAI format ::: +**[LiteLLM Proxy](../simple_proxy) is OpenAI compatible**, you just need the `openai/` prefix before the model + ## Required Variables ```python @@ -87,4 +86,4 @@ for chunk in response: ## **Usage with Langchain, LLamaindex, OpenAI Js, Anthropic SDK, Instructor** -[Follow this doc to see how to use litellm proxy with langchain, llamaindex, anthropic etc](../proxy/user_keys) \ No newline at end of file +#### [Follow this doc to see how to use litellm proxy with langchain, llamaindex, anthropic etc](../proxy/user_keys) \ No newline at end of file diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index af037ea77..18b6a3605 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -810,6 +810,9 @@ print(result) +## Using with Vertex, Boto3, Anthropic SDK (Native format) + +👉 **[Here's how to use litellm proxy with Vertex, boto3, Anthropic SDK - in the native format](../pass_through/vertex_ai.md)** ## Advanced From b2bac2bc345c757ef1df1b865d54573d3506c303 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 11:06:46 -0700 Subject: [PATCH 05/48] fix link on getting started --- docs/my-website/docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 0712c3034..008972046 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -14,7 +14,7 @@ https://github.com/BerriAI/litellm ## How to use LiteLLM You can use litellm through either: -1. [LiteLLM Proxy Server](#openai-proxy) - Server (LLM Gateway) to call 100+ LLMs, load balance, cost tracking across projects +1. [LiteLLM Proxy Server](#litellm-proxy-server-llm-gateway) - Server (LLM Gateway) to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking ### **When to use LiteLLM Proxy Server (LLM Gateway)** From b9d1296319c444c9c667dc0e265a0cb5959b948b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 11:23:45 -0700 Subject: [PATCH 06/48] feat(utils.py): support gemini/vertex ai streaming function param usage --- litellm/tests/test_streaming.py | 40 +++++++++++++++++++++++++-------- litellm/utils.py | 26 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index e310092bd..7d099afc6 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -755,27 +755,40 @@ async def test_completion_gemini_stream(sync_mode): try: litellm.set_verbose = True print("Streaming gemini response") - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, + function1 = [ { - "role": "user", - "content": "Who was Alexander?", - }, + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } ] + messages = [{"role": "user", "content": "What is the weather like in Boston?"}] print("testing gemini streaming") complete_response = "" # Add any assertions here to check the response non_empty_chunks = 0 - + chunks = [] if sync_mode: response = completion( model="gemini/gemini-1.5-flash", messages=messages, stream=True, + functions=function1, ) for idx, chunk in enumerate(response): print(chunk) + chunks.append(chunk) # print(chunk.choices[0].delta) chunk, finished = streaming_format_tests(idx, chunk) if finished: @@ -787,11 +800,13 @@ async def test_completion_gemini_stream(sync_mode): model="gemini/gemini-1.5-flash", messages=messages, stream=True, + functions=function1, ) idx = 0 async for chunk in response: print(chunk) + chunks.append(chunk) # print(chunk.choices[0].delta) chunk, finished = streaming_format_tests(idx, chunk) if finished: @@ -800,10 +815,17 @@ async def test_completion_gemini_stream(sync_mode): complete_response += chunk idx += 1 - if complete_response.strip() == "": - raise Exception("Empty response received") + # if complete_response.strip() == "": + # raise Exception("Empty response received") print(f"completion_response: {complete_response}") - assert non_empty_chunks > 1 + + complete_response = litellm.stream_chunk_builder( + chunks=chunks, messages=messages + ) + + assert complete_response.choices[0].message.function_call is not None + + # assert non_empty_chunks > 1 except litellm.InternalServerError as e: pass except litellm.RateLimitError as e: diff --git a/litellm/utils.py b/litellm/utils.py index 61beabe95..b386a54ec 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8771,6 +8771,7 @@ class CustomStreamWrapper: self.chunks: List = ( [] ) # keep track of the returned chunks - used for calculating the input/output tokens for stream options + self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) def __iter__(self): return self @@ -8778,6 +8779,19 @@ class CustomStreamWrapper: def __aiter__(self): return self + def check_is_function_call(self, logging_obj) -> bool: + if hasattr(logging_obj, "optional_params") and isinstance( + logging_obj.optional_params, dict + ): + if ( + "litellm_param_is_function_call" in logging_obj.optional_params + and logging_obj.optional_params["litellm_param_is_function_call"] + is not None + ): + return True + + return False + def process_chunk(self, chunk: str): """ NLP Cloud streaming returns the entire response, for each chunk. Process this, to only return the delta. @@ -10275,6 +10289,12 @@ class CustomStreamWrapper: ## CHECK FOR TOOL USE if "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0: + if self.is_function_call is True: # user passed in 'functions' param + completion_obj["function_call"] = completion_obj["tool_calls"][0][ + "function" + ] + completion_obj["tool_calls"] = None + self.tool_call = True ## RETURN ARG @@ -10286,8 +10306,13 @@ class CustomStreamWrapper: ) or ( "tool_calls" in completion_obj + and completion_obj["tool_calls"] is not None and len(completion_obj["tool_calls"]) > 0 ) + or ( + "function_call" in completion_obj + and completion_obj["function_call"] is not None + ) ): # cannot set content of an OpenAI Object to be an empty string self.safety_checker() hold, model_response_str = self.check_special_tokens( @@ -10347,6 +10372,7 @@ class CustomStreamWrapper: if self.sent_first_chunk is False: completion_obj["role"] = "assistant" self.sent_first_chunk = True + model_response.choices[0].delta = Delta(**completion_obj) if completion_obj.get("index") is not None: model_response.choices[0].index = completion_obj.get( From 8695cf186d346944f2159525e305cd79452de2ee Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 11:44:37 -0700 Subject: [PATCH 07/48] fix(main.py): fix linting errors --- litellm/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 2cf836890..28c79b4e5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2008,7 +2008,7 @@ def completion( model_response=model_response, print_verbose=print_verbose, optional_params=new_params, - litellm_params=litellm_params, + litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, encoding=encoding, vertex_location=vertex_ai_location, @@ -2095,7 +2095,7 @@ def completion( model_response=model_response, print_verbose=print_verbose, optional_params=new_params, - litellm_params=litellm_params, + litellm_params=litellm_params, # type: ignore logger_fn=logger_fn, encoding=encoding, vertex_location=vertex_ai_location, From 1cbf851ac227694772d6ba75839b02fc2dcbd6e5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 12:04:56 -0700 Subject: [PATCH 08/48] fix(utils.py): fix value check --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index b386a54ec..165d5c093 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8786,7 +8786,7 @@ class CustomStreamWrapper: if ( "litellm_param_is_function_call" in logging_obj.optional_params and logging_obj.optional_params["litellm_param_is_function_call"] - is not None + is True ): return True From 2b40f2eaedd4f68b148fb5fe975d66629a7f7db8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 12:17:48 -0700 Subject: [PATCH 09/48] test(test_function_calling.py): fix test --- litellm/tests/test_function_calling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/tests/test_function_calling.py b/litellm/tests/test_function_calling.py index 6bd0c42cf..81647685f 100644 --- a/litellm/tests/test_function_calling.py +++ b/litellm/tests/test_function_calling.py @@ -142,6 +142,8 @@ def test_parallel_function_call(model): drop_params=True, ) # get a new response from the model where it can see the function response print("second response\n", second_response) + except litellm.InternalServerError: + pass except litellm.RateLimitError: pass except Exception as e: From bea6fb2375b68529f40f7660da7b79e90e701f0f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 13:08:28 -0700 Subject: [PATCH 10/48] fix(vertex_httpx.py): use special param --- litellm/llms/vertex_httpx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 1e842ce57..2f8a27ffe 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -648,7 +648,7 @@ class VertexGeminiConfig: optional_params["presence_penalty"] = value if (param == "tools" or param == "functions") and isinstance(value, list): optional_params["tools"] = self._map_function(value=value) - optional_params["is_function_call"] = ( + optional_params["litellm_param_is_function_call"] = ( True if param == "functions" else False ) if param == "tool_choice" and ( From 3d11b2172681526482693ed751802f7f52b9ba03 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 13:10:04 -0700 Subject: [PATCH 11/48] add fine tuned vertex model support --- litellm/llms/vertex_httpx.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index faecdd790..5042d0f77 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -1161,6 +1161,15 @@ class VertexLLM(BaseLLM): else: url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + # if model is only numeric chars then it's a fine tuned gemini model + # model = 4965075652664360960 + # send to this url: url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + if model.isdigit(): + # It's a fine-tuned Gemini model + url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + if stream is True: + url += "?alt=sse" + if ( api_base is not None ): # for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317 From 07a45fc844ca07d3f3e4c581a397120ad9e42ae1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 13:26:56 -0700 Subject: [PATCH 12/48] add test for test_completion_fine_tuned_model --- .../tests/test_amazing_vertex_completion.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 051b15b99..9a8ce4846 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -2115,3 +2115,87 @@ def test_get_token_url(): assert "/v1/" in url pass + + +@pytest.mark.asyncio +async def test_completion_fine_tuned_model(): + # load_vertex_ai_credentials() + mock_response = AsyncMock() + + def return_val(): + return { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "A canvas vast, a boundless blue,\nWhere clouds paint tales and winds imbue.\nThe sun descends in fiery hue,\nStars shimmer bright, a gentle few.\n\nThe moon ascends, a pearl of light,\nGuiding travelers through the night.\nThe sky embraces, holds all tight,\nA tapestry of wonder, bright." + } + ], + }, + "finishReason": "STOP", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.028930664, + "severity": "HARM_SEVERITY_NEGLIGIBLE", + "severityScore": 0.041992188, + }, + # ... other safety ratings ... + ], + "avgLogprobs": -0.95772853367765187, + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 71, + "totalTokenCount": 78, + }, + } + + mock_response.json = return_val + mock_response.status_code = 200 + + expected_payload = { + "contents": [ + {"role": "user", "parts": [{"text": "Write a short poem about the sky"}]} + ], + "generationConfig": {}, + } + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=mock_response, + ) as mock_post: + # Act: Call the litellm.completion function + response = await litellm.acompletion( + model="vertex_ai_beta/4965075652664360960", + messages=[{"role": "user", "content": "Write a short poem about the sky"}], + ) + + # Assert + mock_post.assert_called_once() + url, kwargs = mock_post.call_args + print("url = ", url) + + # this is the fine-tuned model endpoint + assert ( + url[0] + == "https://us-central1-aiplatform.googleapis.com/v1/projects/adroit-crow-413218/locations/us-central1/endpoints/4965075652664360960:generateContent" + ) + + print("call args = ", kwargs) + args_to_vertexai = kwargs["json"] + + print("args to vertex ai call:", args_to_vertexai) + + assert args_to_vertexai == expected_payload + assert response.choices[0].message.content.startswith("A canvas vast") + assert response.choices[0].finish_reason == "stop" + assert response.usage.total_tokens == 78 + + # Optional: Print for debugging + print("Arguments passed to Vertex AI:", args_to_vertexai) + print("Response:", response) From 8233a20db0207f4520201f72f2485a9804a60225 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 13:28:25 -0700 Subject: [PATCH 13/48] docs: fix dead links --- docs/my-website/docs/proxy/guardrails/aporia_api.md | 2 +- docs/my-website/docs/proxy/guardrails/bedrock.md | 2 +- docs/my-website/docs/proxy/guardrails/custom_guardrail.md | 6 +++--- docs/my-website/docs/proxy/guardrails/lakera_ai.md | 2 +- docs/my-website/docs/proxy/guardrails/quick_start.md | 2 +- docs/my-website/docs/tutorials/litellm_proxy_aporia.md | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/aporia_api.md b/docs/my-website/docs/proxy/guardrails/aporia_api.md index d5b6bd636..d45c34d47 100644 --- a/docs/my-website/docs/proxy/guardrails/aporia_api.md +++ b/docs/my-website/docs/proxy/guardrails/aporia_api.md @@ -71,7 +71,7 @@ litellm --config config.yaml --detailed_debug ## 4. Test request -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** diff --git a/docs/my-website/docs/proxy/guardrails/bedrock.md b/docs/my-website/docs/proxy/guardrails/bedrock.md index ac8aa1c1b..84e17ba86 100644 --- a/docs/my-website/docs/proxy/guardrails/bedrock.md +++ b/docs/my-website/docs/proxy/guardrails/bedrock.md @@ -40,7 +40,7 @@ litellm --config config.yaml --detailed_debug ### 3. Test request -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index 57d292d42..469196d86 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -202,7 +202,7 @@ litellm --config config.yaml --detailed_debug #### Test `"custom-pre-guard"` -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** @@ -282,7 +282,7 @@ curl -i http://localhost:4000/v1/chat/completions \ #### Test `"custom-during-guard"` -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** @@ -346,7 +346,7 @@ curl -i http://localhost:4000/v1/chat/completions \ -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** diff --git a/docs/my-website/docs/proxy/guardrails/lakera_ai.md b/docs/my-website/docs/proxy/guardrails/lakera_ai.md index 2c6dd2b42..ba1ca0b21 100644 --- a/docs/my-website/docs/proxy/guardrails/lakera_ai.md +++ b/docs/my-website/docs/proxy/guardrails/lakera_ai.md @@ -46,7 +46,7 @@ litellm --config config.yaml --detailed_debug ### 3. Test request -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** diff --git a/docs/my-website/docs/proxy/guardrails/quick_start.md b/docs/my-website/docs/proxy/guardrails/quick_start.md index 30f5051d2..10a078fed 100644 --- a/docs/my-website/docs/proxy/guardrails/quick_start.md +++ b/docs/my-website/docs/proxy/guardrails/quick_start.md @@ -48,7 +48,7 @@ litellm --config config.yaml --detailed_debug ## 3. Test request -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** diff --git a/docs/my-website/docs/tutorials/litellm_proxy_aporia.md b/docs/my-website/docs/tutorials/litellm_proxy_aporia.md index 1fea3037f..3b5bada2b 100644 --- a/docs/my-website/docs/tutorials/litellm_proxy_aporia.md +++ b/docs/my-website/docs/tutorials/litellm_proxy_aporia.md @@ -72,7 +72,7 @@ litellm --config config.yaml --detailed_debug ## 4. Test request -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys##request-format)** +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** From 8d9f94ede7a4d26d3c4262afb4d8d5e9d34d8507 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 13:39:20 -0700 Subject: [PATCH 14/48] vertex add finetuned models --- docs/my-website/docs/providers/vertex.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 967f45b84..ffd00d9f5 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1194,6 +1194,14 @@ response = completion( |------------------|--------------------------------------| | gemini-pro | `completion('gemini-pro', messages)`, `completion('vertex_ai/gemini-pro', messages)` | +## Fine-tuned Models + +Fine tuned models on vertex have a numerical model/endpoint id. + +| Model Name | Function Call | +|------------------|--------------------------------------| +| your fine tuned model | `completion(model='vertex_ai/4965075652664360960', messages)`| + ## Gemini Pro Vision | Model Name | Function Call | |------------------|--------------------------------------| From 174b1c43e35c6ecb617c5d95de0569937cb850f5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 13:59:57 -0700 Subject: [PATCH 15/48] fix(utils.py): support 'PERPLEXITY_API_KEY' in env --- litellm/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 61beabe95..68eac7c3a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4610,7 +4610,11 @@ def get_llm_provider( if custom_llm_provider == "perplexity": # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai api_base = api_base or get_secret("PERPLEXITY_API_BASE") or "https://api.perplexity.ai" # type: ignore - dynamic_api_key = api_key or get_secret("PERPLEXITYAI_API_KEY") + dynamic_api_key = ( + api_key + or get_secret("PERPLEXITYAI_API_KEY") + or get_secret("PERPLEXITY_API_KEY") + ) elif custom_llm_provider == "anyscale": # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = api_base or get_secret("ANYSCALE_API_BASE") or "https://api.endpoints.anyscale.com/v1" # type: ignore @@ -6647,7 +6651,6 @@ def exception_type( if message is not None and isinstance(message, str): message = message.replace("OPENAI", custom_llm_provider.upper()) - message = message.replace("openai", custom_llm_provider) message = message.replace("OpenAI", custom_llm_provider) if custom_llm_provider == "openai": exception_provider = "OpenAI" + "Exception" From f9ea0d8fa985b97793444adb47dabff33a06b94a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 14:16:25 -0700 Subject: [PATCH 16/48] refactor cohere to be in a folder --- litellm/llms/{cohere_chat.py => cohere/chat.py} | 2 +- litellm/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename litellm/llms/{cohere_chat.py => cohere/chat.py} (99%) diff --git a/litellm/llms/cohere_chat.py b/litellm/llms/cohere/chat.py similarity index 99% rename from litellm/llms/cohere_chat.py rename to litellm/llms/cohere/chat.py index f13e74614..b2569b429 100644 --- a/litellm/llms/cohere_chat.py +++ b/litellm/llms/cohere/chat.py @@ -13,7 +13,7 @@ import litellm from litellm.types.llms.cohere import ToolResultObject from litellm.utils import Choices, Message, ModelResponse, Usage -from .prompt_templates.factory import cohere_message_pt, cohere_messages_pt_v2 +from ..prompt_templates.factory import cohere_message_pt, cohere_messages_pt_v2 class CohereError(Exception): diff --git a/litellm/main.py b/litellm/main.py index 2cf836890..1772e9b2b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -83,7 +83,6 @@ from .llms import ( clarifai, cloudflare, cohere, - cohere_chat, gemini, huggingface_restapi, maritalk, @@ -107,6 +106,7 @@ from .llms.anthropic_text import AnthropicTextCompletion from .llms.azure import AzureChatCompletion, _check_dynamic_azure_params from .llms.azure_text import AzureTextCompletion from .llms.bedrock_httpx import BedrockConverseLLM, BedrockLLM +from .llms.cohere import chat as cohere_chat from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks import DatabricksChatCompletion from .llms.huggingface_restapi import Huggingface From da63775371f39d27ae3b832cd502ccbeda832b18 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 14:28:50 -0700 Subject: [PATCH 17/48] use common folder for cohere --- litellm/__init__.py | 2 +- litellm/llms/cohere.py | 414 ------------------------------ litellm/llms/cohere/completion.py | 253 ++++++++++++++++++ litellm/llms/cohere/embed.py | 201 +++++++++++++++ litellm/main.py | 6 +- 5 files changed, 459 insertions(+), 417 deletions(-) create mode 100644 litellm/llms/cohere/completion.py create mode 100644 litellm/llms/cohere/embed.py diff --git a/litellm/__init__.py b/litellm/__init__.py index bda12e5dd..207ea3ebd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -838,7 +838,7 @@ from .llms.databricks import DatabricksConfig, DatabricksEmbeddingConfig from .llms.predibase import PredibaseConfig from .llms.anthropic_text import AnthropicTextConfig from .llms.replicate import ReplicateConfig -from .llms.cohere import CohereConfig +from .llms.cohere.completion import CohereConfig from .llms.clarifai import ClarifaiConfig from .llms.ai21 import AI21Config from .llms.together_ai import TogetherAIConfig diff --git a/litellm/llms/cohere.py b/litellm/llms/cohere.py index 8bd1051e8..e69de29bb 100644 --- a/litellm/llms/cohere.py +++ b/litellm/llms/cohere.py @@ -1,414 +0,0 @@ -#################### OLD ######################## -##### See `cohere_chat.py` for `/chat` calls #### -################################################# -import json -import os -import time -import traceback -import types -from enum import Enum -from typing import Any, Callable, Optional, Union - -import httpx # type: ignore -import requests # type: ignore - -import litellm -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.utils import Choices, Message, ModelResponse, Usage - - -class CohereError(Exception): - def __init__(self, status_code, message): - self.status_code = status_code - self.message = message - self.request = httpx.Request( - method="POST", url="https://api.cohere.ai/v1/generate" - ) - self.response = httpx.Response(status_code=status_code, request=self.request) - super().__init__( - self.message - ) # Call the base class constructor with the parameters it needs - - -def construct_cohere_tool(tools=None): - if tools is None: - tools = [] - return {"tools": tools} - - -class CohereConfig: - """ - Reference: https://docs.cohere.com/reference/generate - - The class `CohereConfig` provides configuration for the Cohere's API interface. Below are the parameters: - - - `num_generations` (integer): Maximum number of generations returned. Default is 1, with a minimum value of 1 and a maximum value of 5. - - - `max_tokens` (integer): Maximum number of tokens the model will generate as part of the response. Default value is 20. - - - `truncate` (string): Specifies how the API handles inputs longer than maximum token length. Options include NONE, START, END. Default is END. - - - `temperature` (number): A non-negative float controlling the randomness in generation. Lower temperatures result in less random generations. Default is 0.75. - - - `preset` (string): Identifier of a custom preset, a combination of parameters such as prompt, temperature etc. - - - `end_sequences` (array of strings): The generated text gets cut at the beginning of the earliest occurrence of an end sequence, which will be excluded from the text. - - - `stop_sequences` (array of strings): The generated text gets cut at the end of the earliest occurrence of a stop sequence, which will be included in the text. - - - `k` (integer): Limits generation at each step to top `k` most likely tokens. Default is 0. - - - `p` (number): Limits generation at each step to most likely tokens with total probability mass of `p`. Default is 0. - - - `frequency_penalty` (number): Reduces repetitiveness of generated tokens. Higher values apply stronger penalties to previously occurred tokens. - - - `presence_penalty` (number): Reduces repetitiveness of generated tokens. Similar to frequency_penalty, but this penalty applies equally to all tokens that have already appeared. - - - `return_likelihoods` (string): Specifies how and if token likelihoods are returned with the response. Options include GENERATION, ALL and NONE. - - - `logit_bias` (object): Used to prevent the model from generating unwanted tokens or to incentivize it to include desired tokens. e.g. {"hello_world": 1233} - """ - - num_generations: Optional[int] = None - max_tokens: Optional[int] = None - truncate: Optional[str] = None - temperature: Optional[int] = None - preset: Optional[str] = None - end_sequences: Optional[list] = None - stop_sequences: Optional[list] = None - k: Optional[int] = None - p: Optional[int] = None - frequency_penalty: Optional[int] = None - presence_penalty: Optional[int] = None - return_likelihoods: Optional[str] = None - logit_bias: Optional[dict] = None - - def __init__( - self, - num_generations: Optional[int] = None, - max_tokens: Optional[int] = None, - truncate: Optional[str] = None, - temperature: Optional[int] = None, - preset: Optional[str] = None, - end_sequences: Optional[list] = None, - stop_sequences: Optional[list] = None, - k: Optional[int] = None, - p: Optional[int] = None, - frequency_penalty: Optional[int] = None, - presence_penalty: Optional[int] = None, - return_likelihoods: Optional[str] = None, - logit_bias: Optional[dict] = None, - ) -> None: - locals_ = locals() - for key, value in locals_.items(): - if key != "self" and value is not None: - setattr(self.__class__, key, value) - - @classmethod - def get_config(cls): - return { - k: v - for k, v in cls.__dict__.items() - if not k.startswith("__") - and not isinstance( - v, - ( - types.FunctionType, - types.BuiltinFunctionType, - classmethod, - staticmethod, - ), - ) - and v is not None - } - - -def validate_environment(api_key, headers: dict): - headers.update( - { - "Request-Source": "unspecified:litellm", - "accept": "application/json", - "content-type": "application/json", - } - ) - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - return headers - - -def completion( - model: str, - messages: list, - api_base: str, - model_response: ModelResponse, - print_verbose: Callable, - encoding, - api_key, - logging_obj, - headers: dict, - optional_params=None, - litellm_params=None, - logger_fn=None, -): - headers = validate_environment(api_key, headers=headers) - completion_url = api_base - model = model - prompt = " ".join(message["content"] for message in messages) - - ## Load Config - config = litellm.CohereConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## Handle Tool Calling - if "tools" in optional_params: - _is_function_call = True - tool_calling_system_prompt = construct_cohere_tool( - tools=optional_params["tools"] - ) - optional_params["tools"] = tool_calling_system_prompt - - data = { - "model": model, - "prompt": prompt, - **optional_params, - } - - ## LOGGING - logging_obj.pre_call( - input=prompt, - api_key=api_key, - additional_args={ - "complete_input_dict": data, - "headers": headers, - "api_base": completion_url, - }, - ) - ## COMPLETION CALL - response = requests.post( - completion_url, - headers=headers, - data=json.dumps(data), - stream=optional_params["stream"] if "stream" in optional_params else False, - ) - ## error handling for cohere calls - if response.status_code != 200: - raise CohereError(message=response.text, status_code=response.status_code) - - if "stream" in optional_params and optional_params["stream"] == True: - return response.iter_lines() - else: - ## LOGGING - logging_obj.post_call( - input=prompt, - api_key=api_key, - original_response=response.text, - additional_args={"complete_input_dict": data}, - ) - print_verbose(f"raw model_response: {response.text}") - ## RESPONSE OBJECT - completion_response = response.json() - if "error" in completion_response: - raise CohereError( - message=completion_response["error"], - status_code=response.status_code, - ) - else: - try: - choices_list = [] - for idx, item in enumerate(completion_response["generations"]): - if len(item["text"]) > 0: - message_obj = Message(content=item["text"]) - else: - message_obj = Message(content=None) - choice_obj = Choices( - finish_reason=item["finish_reason"], - index=idx + 1, - message=message_obj, - ) - choices_list.append(choice_obj) - model_response.choices = choices_list # type: ignore - except Exception as e: - raise CohereError( - message=response.text, status_code=response.status_code - ) - - ## CALCULATING USAGE - prompt_tokens = len(encoding.encode(prompt)) - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) - ) - - model_response.created = int(time.time()) - model_response.model = model - usage = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - setattr(model_response, "usage", usage) - return model_response - - -def _process_embedding_response( - embeddings: list, - model_response: litellm.EmbeddingResponse, - model: str, - encoding: Any, - input: list, -) -> litellm.EmbeddingResponse: - output_data = [] - for idx, embedding in enumerate(embeddings): - output_data.append( - {"object": "embedding", "index": idx, "embedding": embedding} - ) - model_response.object = "list" - model_response.data = output_data - model_response.model = model - input_tokens = 0 - for text in input: - input_tokens += len(encoding.encode(text)) - - setattr( - model_response, - "usage", - Usage( - prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens - ), - ) - - return model_response - - -async def async_embedding( - model: str, - data: dict, - input: list, - model_response: litellm.utils.EmbeddingResponse, - timeout: Union[float, httpx.Timeout], - logging_obj: LiteLLMLoggingObj, - optional_params: dict, - api_base: str, - api_key: Optional[str], - headers: dict, - encoding: Callable, - client: Optional[AsyncHTTPHandler] = None, -): - - ## LOGGING - logging_obj.pre_call( - input=input, - api_key=api_key, - additional_args={ - "complete_input_dict": data, - "headers": headers, - "api_base": api_base, - }, - ) - ## COMPLETION CALL - if client is None: - client = AsyncHTTPHandler(concurrent_limit=1) - - response = await client.post(api_base, headers=headers, data=json.dumps(data)) - - ## LOGGING - logging_obj.post_call( - input=input, - api_key=api_key, - additional_args={"complete_input_dict": data}, - original_response=response, - ) - - embeddings = response.json()["embeddings"] - - ## PROCESS RESPONSE ## - return _process_embedding_response( - embeddings=embeddings, - model_response=model_response, - model=model, - encoding=encoding, - input=input, - ) - - -def embedding( - model: str, - input: list, - model_response: litellm.EmbeddingResponse, - logging_obj: LiteLLMLoggingObj, - optional_params: dict, - headers: dict, - encoding: Any, - api_key: Optional[str] = None, - aembedding: Optional[bool] = None, - timeout: Union[float, httpx.Timeout] = httpx.Timeout(None), - client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, -): - headers = validate_environment(api_key, headers=headers) - embed_url = "https://api.cohere.ai/v1/embed" - model = model - data = {"model": model, "texts": input, **optional_params} - - if "3" in model and "input_type" not in data: - # cohere v3 embedding models require input_type, if no input_type is provided, default to "search_document" - data["input_type"] = "search_document" - - ## LOGGING - logging_obj.pre_call( - input=input, - api_key=api_key, - additional_args={"complete_input_dict": data}, - ) - - ## ROUTING - if aembedding is True: - return async_embedding( - model=model, - data=data, - input=input, - model_response=model_response, - timeout=timeout, - logging_obj=logging_obj, - optional_params=optional_params, - api_base=embed_url, - api_key=api_key, - headers=headers, - encoding=encoding, - ) - ## COMPLETION CALL - if client is None or not isinstance(client, HTTPHandler): - client = HTTPHandler(concurrent_limit=1) - response = client.post(embed_url, headers=headers, data=json.dumps(data)) - ## LOGGING - logging_obj.post_call( - input=input, - api_key=api_key, - additional_args={"complete_input_dict": data}, - original_response=response, - ) - """ - response - { - 'object': "list", - 'data': [ - - ] - 'model', - 'usage' - } - """ - if response.status_code != 200: - raise CohereError(message=response.text, status_code=response.status_code) - embeddings = response.json()["embeddings"] - - return _process_embedding_response( - embeddings=embeddings, - model_response=model_response, - model=model, - encoding=encoding, - input=input, - ) diff --git a/litellm/llms/cohere/completion.py b/litellm/llms/cohere/completion.py new file mode 100644 index 000000000..3e8bd4ded --- /dev/null +++ b/litellm/llms/cohere/completion.py @@ -0,0 +1,253 @@ +##### Calls /generate endpoint ####### + +import json +import os +import time +import traceback +import types +from enum import Enum +from typing import Any, Callable, Optional, Union + +import httpx # type: ignore +import requests # type: ignore + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.utils import Choices, Message, ModelResponse, Usage + + +class CohereError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", url="https://api.cohere.ai/v1/generate" + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +def construct_cohere_tool(tools=None): + if tools is None: + tools = [] + return {"tools": tools} + + +class CohereConfig: + """ + Reference: https://docs.cohere.com/reference/generate + + The class `CohereConfig` provides configuration for the Cohere's API interface. Below are the parameters: + + - `num_generations` (integer): Maximum number of generations returned. Default is 1, with a minimum value of 1 and a maximum value of 5. + + - `max_tokens` (integer): Maximum number of tokens the model will generate as part of the response. Default value is 20. + + - `truncate` (string): Specifies how the API handles inputs longer than maximum token length. Options include NONE, START, END. Default is END. + + - `temperature` (number): A non-negative float controlling the randomness in generation. Lower temperatures result in less random generations. Default is 0.75. + + - `preset` (string): Identifier of a custom preset, a combination of parameters such as prompt, temperature etc. + + - `end_sequences` (array of strings): The generated text gets cut at the beginning of the earliest occurrence of an end sequence, which will be excluded from the text. + + - `stop_sequences` (array of strings): The generated text gets cut at the end of the earliest occurrence of a stop sequence, which will be included in the text. + + - `k` (integer): Limits generation at each step to top `k` most likely tokens. Default is 0. + + - `p` (number): Limits generation at each step to most likely tokens with total probability mass of `p`. Default is 0. + + - `frequency_penalty` (number): Reduces repetitiveness of generated tokens. Higher values apply stronger penalties to previously occurred tokens. + + - `presence_penalty` (number): Reduces repetitiveness of generated tokens. Similar to frequency_penalty, but this penalty applies equally to all tokens that have already appeared. + + - `return_likelihoods` (string): Specifies how and if token likelihoods are returned with the response. Options include GENERATION, ALL and NONE. + + - `logit_bias` (object): Used to prevent the model from generating unwanted tokens or to incentivize it to include desired tokens. e.g. {"hello_world": 1233} + """ + + num_generations: Optional[int] = None + max_tokens: Optional[int] = None + truncate: Optional[str] = None + temperature: Optional[int] = None + preset: Optional[str] = None + end_sequences: Optional[list] = None + stop_sequences: Optional[list] = None + k: Optional[int] = None + p: Optional[int] = None + frequency_penalty: Optional[int] = None + presence_penalty: Optional[int] = None + return_likelihoods: Optional[str] = None + logit_bias: Optional[dict] = None + + def __init__( + self, + num_generations: Optional[int] = None, + max_tokens: Optional[int] = None, + truncate: Optional[str] = None, + temperature: Optional[int] = None, + preset: Optional[str] = None, + end_sequences: Optional[list] = None, + stop_sequences: Optional[list] = None, + k: Optional[int] = None, + p: Optional[int] = None, + frequency_penalty: Optional[int] = None, + presence_penalty: Optional[int] = None, + return_likelihoods: Optional[str] = None, + logit_bias: Optional[dict] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + +def validate_environment(api_key, headers: dict): + headers.update( + { + "Request-Source": "unspecified:litellm", + "accept": "application/json", + "content-type": "application/json", + } + ) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + +def completion( + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + headers: dict, + optional_params=None, + litellm_params=None, + logger_fn=None, +): + headers = validate_environment(api_key, headers=headers) + completion_url = api_base + model = model + prompt = " ".join(message["content"] for message in messages) + + ## Load Config + config = litellm.CohereConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## Handle Tool Calling + if "tools" in optional_params: + _is_function_call = True + tool_calling_system_prompt = construct_cohere_tool( + tools=optional_params["tools"] + ) + optional_params["tools"] = tool_calling_system_prompt + + data = { + "model": model, + "prompt": prompt, + **optional_params, + } + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": completion_url, + }, + ) + ## COMPLETION CALL + response = requests.post( + completion_url, + headers=headers, + data=json.dumps(data), + stream=optional_params["stream"] if "stream" in optional_params else False, + ) + ## error handling for cohere calls + if response.status_code != 200: + raise CohereError(message=response.text, status_code=response.status_code) + + if "stream" in optional_params and optional_params["stream"] == True: + return response.iter_lines() + else: + ## LOGGING + logging_obj.post_call( + input=prompt, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + ## RESPONSE OBJECT + completion_response = response.json() + if "error" in completion_response: + raise CohereError( + message=completion_response["error"], + status_code=response.status_code, + ) + else: + try: + choices_list = [] + for idx, item in enumerate(completion_response["generations"]): + if len(item["text"]) > 0: + message_obj = Message(content=item["text"]) + else: + message_obj = Message(content=None) + choice_obj = Choices( + finish_reason=item["finish_reason"], + index=idx + 1, + message=message_obj, + ) + choices_list.append(choice_obj) + model_response.choices = choices_list # type: ignore + except Exception as e: + raise CohereError( + message=response.text, status_code=response.status_code + ) + + ## CALCULATING USAGE + prompt_tokens = len(encoding.encode(prompt)) + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content", "")) + ) + + model_response.created = int(time.time()) + model_response.model = model + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + setattr(model_response, "usage", usage) + return model_response diff --git a/litellm/llms/cohere/embed.py b/litellm/llms/cohere/embed.py new file mode 100644 index 000000000..81c84c422 --- /dev/null +++ b/litellm/llms/cohere/embed.py @@ -0,0 +1,201 @@ +import json +import os +import time +import traceback +import types +from enum import Enum +from typing import Any, Callable, Optional, Union + +import httpx # type: ignore +import requests # type: ignore + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.utils import Choices, Message, ModelResponse, Usage + + +def validate_environment(api_key, headers: dict): + headers.update( + { + "Request-Source": "unspecified:litellm", + "accept": "application/json", + "content-type": "application/json", + } + ) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + +class CohereError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", url="https://api.cohere.ai/v1/generate" + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +def _process_embedding_response( + embeddings: list, + model_response: litellm.EmbeddingResponse, + model: str, + encoding: Any, + input: list, +) -> litellm.EmbeddingResponse: + output_data = [] + for idx, embedding in enumerate(embeddings): + output_data.append( + {"object": "embedding", "index": idx, "embedding": embedding} + ) + model_response.object = "list" + model_response.data = output_data + model_response.model = model + input_tokens = 0 + for text in input: + input_tokens += len(encoding.encode(text)) + + setattr( + model_response, + "usage", + Usage( + prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens + ), + ) + + return model_response + + +async def async_embedding( + model: str, + data: dict, + input: list, + model_response: litellm.utils.EmbeddingResponse, + timeout: Union[float, httpx.Timeout], + logging_obj: LiteLLMLoggingObj, + optional_params: dict, + api_base: str, + api_key: Optional[str], + headers: dict, + encoding: Callable, + client: Optional[AsyncHTTPHandler] = None, +): + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": api_base, + }, + ) + ## COMPLETION CALL + if client is None: + client = AsyncHTTPHandler(concurrent_limit=1) + + response = await client.post(api_base, headers=headers, data=json.dumps(data)) + + ## LOGGING + logging_obj.post_call( + input=input, + api_key=api_key, + additional_args={"complete_input_dict": data}, + original_response=response, + ) + + embeddings = response.json()["embeddings"] + + ## PROCESS RESPONSE ## + return _process_embedding_response( + embeddings=embeddings, + model_response=model_response, + model=model, + encoding=encoding, + input=input, + ) + + +def embedding( + model: str, + input: list, + model_response: litellm.EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + optional_params: dict, + headers: dict, + encoding: Any, + api_key: Optional[str] = None, + aembedding: Optional[bool] = None, + timeout: Union[float, httpx.Timeout] = httpx.Timeout(None), + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, +): + headers = validate_environment(api_key, headers=headers) + embed_url = "https://api.cohere.ai/v1/embed" + model = model + data = {"model": model, "texts": input, **optional_params} + + if "3" in model and "input_type" not in data: + # cohere v3 embedding models require input_type, if no input_type is provided, default to "search_document" + data["input_type"] = "search_document" + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key=api_key, + additional_args={"complete_input_dict": data}, + ) + + ## ROUTING + if aembedding is True: + return async_embedding( + model=model, + data=data, + input=input, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + optional_params=optional_params, + api_base=embed_url, + api_key=api_key, + headers=headers, + encoding=encoding, + ) + ## COMPLETION CALL + if client is None or not isinstance(client, HTTPHandler): + client = HTTPHandler(concurrent_limit=1) + response = client.post(embed_url, headers=headers, data=json.dumps(data)) + ## LOGGING + logging_obj.post_call( + input=input, + api_key=api_key, + additional_args={"complete_input_dict": data}, + original_response=response, + ) + """ + response + { + 'object': "list", + 'data': [ + + ] + 'model', + 'usage' + } + """ + if response.status_code != 200: + raise CohereError(message=response.text, status_code=response.status_code) + embeddings = response.json()["embeddings"] + + return _process_embedding_response( + embeddings=embeddings, + model_response=model_response, + model=model, + encoding=encoding, + input=input, + ) diff --git a/litellm/main.py b/litellm/main.py index 1772e9b2b..dd6a9e1b6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -107,6 +107,8 @@ from .llms.azure import AzureChatCompletion, _check_dynamic_azure_params from .llms.azure_text import AzureTextCompletion from .llms.bedrock_httpx import BedrockConverseLLM, BedrockLLM from .llms.cohere import chat as cohere_chat +from .llms.cohere import completion as cohere_completion +from .llms.cohere import embed as cohere_embed from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks import DatabricksChatCompletion from .llms.huggingface_restapi import Huggingface @@ -1645,7 +1647,7 @@ def completion( if extra_headers is not None: headers.update(extra_headers) - model_response = cohere.completion( + model_response = cohere_completion.completion( model=model, messages=messages, api_base=api_base, @@ -3457,7 +3459,7 @@ def embedding( headers = extra_headers else: headers = {} - response = cohere.embedding( + response = cohere_embed.embedding( model=model, input=input, optional_params=optional_params, From 9a181067452874f028d38b397ec0762b0c737590 Mon Sep 17 00:00:00 2001 From: John HU Date: Mon, 26 Aug 2024 14:41:47 -0700 Subject: [PATCH 18/48] Add pricing for imagen-3 and imagen-3-fast --- model_prices_and_context_window.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0f483b083..4f5fffc52 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2189,6 +2189,18 @@ "mode": "image_generation", "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/imagen-3.0-generate-001": { + "cost_per_image": 0.04, + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + }, + "vertex_ai/imagen-3.0-fast-generate-001": { + "cost_per_image": 0.02, + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + }, "text-embedding-004": { "max_tokens": 3072, "max_input_tokens": 3072, From 8e9acd117bcf8fe14800936c9e8c9ec058d2526c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 15:08:08 -0700 Subject: [PATCH 19/48] fix(sagemaker.py): support streaming for messages api Fixes https://github.com/BerriAI/litellm/issues/5372 --- litellm/__init__.py | 2 +- litellm/litellm_core_utils/streaming_utils.py | 4 +- litellm/llms/databricks.py | 39 ++++++++++--- litellm/llms/{ => sagemaker}/sagemaker.py | 57 +++++++++++++++---- litellm/main.py | 2 +- litellm/tests/test_sagemaker.py | 22 +++++-- litellm/types/llms/openai.py | 8 +++ litellm/types/utils.py | 40 +++++++++++-- 8 files changed, 142 insertions(+), 32 deletions(-) rename litellm/llms/{ => sagemaker}/sagemaker.py (94%) diff --git a/litellm/__init__.py b/litellm/__init__.py index bda12e5dd..6e2309a8c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -856,7 +856,7 @@ from .llms.vertex_httpx import ( from .llms.vertex_ai import VertexAITextEmbeddingConfig from .llms.vertex_ai_anthropic import VertexAIAnthropicConfig from .llms.vertex_ai_partner import VertexAILlama3Config -from .llms.sagemaker import SagemakerConfig +from .llms.sagemaker.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.ollama_chat import OllamaChatConfig from .llms.maritalk import MaritTalkConfig diff --git a/litellm/litellm_core_utils/streaming_utils.py b/litellm/litellm_core_utils/streaming_utils.py index ca8d58e9f..386842fe2 100644 --- a/litellm/litellm_core_utils/streaming_utils.py +++ b/litellm/litellm_core_utils/streaming_utils.py @@ -13,4 +13,6 @@ def generic_chunk_has_all_required_fields(chunk: dict) -> bool: # this is an optional field in GenericStreamingChunk, it's not required to be present _all_fields.pop("provider_specific_fields", None) - return all(key in chunk for key in _all_fields) + decision = all(key in _all_fields for key in chunk) + + return decision diff --git a/litellm/llms/databricks.py b/litellm/llms/databricks.py index bd529046a..9b321cd33 100644 --- a/litellm/llms/databricks.py +++ b/litellm/llms/databricks.py @@ -7,7 +7,7 @@ import time import types from enum import Enum from functools import partial -from typing import Callable, List, Literal, Optional, Tuple, Union +from typing import Any, Callable, List, Literal, Optional, Tuple, Union import httpx # type: ignore import requests # type: ignore @@ -22,7 +22,11 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ChatCompletionUsageBlock, ) -from litellm.types.utils import GenericStreamingChunk, ProviderField +from litellm.types.utils import ( + CustomStreamingDecoder, + GenericStreamingChunk, + ProviderField, +) from litellm.utils import CustomStreamWrapper, EmbeddingResponse, ModelResponse, Usage from .base import BaseLLM @@ -171,15 +175,21 @@ async def make_call( model: str, messages: list, logging_obj, + streaming_decoder: Optional[CustomStreamingDecoder] = None, ): response = await client.post(api_base, headers=headers, data=data, stream=True) if response.status_code != 200: raise DatabricksError(status_code=response.status_code, message=response.text) - completion_stream = ModelResponseIterator( - streaming_response=response.aiter_lines(), sync_stream=False - ) + if streaming_decoder is not None: + completion_stream: Any = streaming_decoder.aiter_bytes( + response.aiter_bytes(chunk_size=1024) + ) + else: + completion_stream = ModelResponseIterator( + streaming_response=response.aiter_lines(), sync_stream=False + ) # LOGGING logging_obj.post_call( input=messages, @@ -199,6 +209,7 @@ def make_sync_call( model: str, messages: list, logging_obj, + streaming_decoder: Optional[CustomStreamingDecoder] = None, ): if client is None: client = HTTPHandler() # Create a new client if none provided @@ -208,9 +219,14 @@ def make_sync_call( if response.status_code != 200: raise DatabricksError(status_code=response.status_code, message=response.read()) - completion_stream = ModelResponseIterator( - streaming_response=response.iter_lines(), sync_stream=True - ) + if streaming_decoder is not None: + completion_stream = streaming_decoder.iter_bytes( + response.iter_bytes(chunk_size=1024) + ) + else: + completion_stream = ModelResponseIterator( + streaming_response=response.iter_lines(), sync_stream=True + ) # LOGGING logging_obj.post_call( @@ -283,6 +299,7 @@ class DatabricksChatCompletion(BaseLLM): logger_fn=None, headers={}, client: Optional[AsyncHTTPHandler] = None, + streaming_decoder: Optional[CustomStreamingDecoder] = None, ) -> CustomStreamWrapper: data["stream"] = True @@ -296,6 +313,7 @@ class DatabricksChatCompletion(BaseLLM): model=model, messages=messages, logging_obj=logging_obj, + streaming_decoder=streaming_decoder, ), model=model, custom_llm_provider=custom_llm_provider, @@ -371,6 +389,9 @@ class DatabricksChatCompletion(BaseLLM): timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, custom_endpoint: Optional[bool] = None, + streaming_decoder: Optional[ + CustomStreamingDecoder + ] = None, # if openai-compatible api needs custom stream decoder - e.g. sagemaker ): custom_endpoint = custom_endpoint or optional_params.pop( "custom_endpoint", None @@ -436,6 +457,7 @@ class DatabricksChatCompletion(BaseLLM): headers=headers, client=client, custom_llm_provider=custom_llm_provider, + streaming_decoder=streaming_decoder, ) else: return self.acompletion_function( @@ -473,6 +495,7 @@ class DatabricksChatCompletion(BaseLLM): model=model, messages=messages, logging_obj=logging_obj, + streaming_decoder=streaming_decoder, ), model=model, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/sagemaker.py b/litellm/llms/sagemaker/sagemaker.py similarity index 94% rename from litellm/llms/sagemaker.py rename to litellm/llms/sagemaker/sagemaker.py index 33be2efb8..32f73f7ee 100644 --- a/litellm/llms/sagemaker.py +++ b/litellm/llms/sagemaker/sagemaker.py @@ -24,8 +24,11 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionUsageBlock, + OpenAIChatCompletionChunk, ) +from litellm.types.utils import CustomStreamingDecoder from litellm.types.utils import GenericStreamingChunk as GChunk +from litellm.types.utils import StreamingChatCompletionChunk from litellm.utils import ( CustomStreamWrapper, EmbeddingResponse, @@ -34,8 +37,8 @@ from litellm.utils import ( get_secret, ) -from .base_aws_llm import BaseAWSLLM -from .prompt_templates.factory import custom_prompt, prompt_factory +from ..base_aws_llm import BaseAWSLLM +from ..prompt_templates.factory import custom_prompt, prompt_factory _response_stream_shape_cache = None @@ -241,6 +244,10 @@ class SagemakerLLM(BaseAWSLLM): aws_region_name=aws_region_name, ) + custom_stream_decoder = AWSEventStreamDecoder( + model="", is_messages_api=True + ) + return openai_like_chat_completions.completion( model=model, messages=messages, @@ -259,6 +266,7 @@ class SagemakerLLM(BaseAWSLLM): headers=prepared_request.headers, custom_endpoint=True, custom_llm_provider="sagemaker_chat", + streaming_decoder=custom_stream_decoder, # type: ignore ) ## Load Config @@ -332,7 +340,7 @@ class SagemakerLLM(BaseAWSLLM): ) return response else: - if stream is not None and stream == True: + if stream is not None and stream is True: sync_handler = _get_httpx_client() sync_response = sync_handler.post( url=prepared_request.url, @@ -847,12 +855,21 @@ def get_response_stream_shape(): class AWSEventStreamDecoder: - def __init__(self, model: str) -> None: + def __init__(self, model: str, is_messages_api: Optional[bool] = None) -> None: from botocore.parsers import EventStreamJSONParser self.model = model self.parser = EventStreamJSONParser() self.content_blocks: List = [] + self.is_messages_api = is_messages_api + + def _chunk_parser_messages_api( + self, chunk_data: dict + ) -> StreamingChatCompletionChunk: + + openai_chunk = StreamingChatCompletionChunk(**chunk_data) + + return openai_chunk def _chunk_parser(self, chunk_data: dict) -> GChunk: verbose_logger.debug("in sagemaker chunk parser, chunk_data %s", chunk_data) @@ -868,6 +885,7 @@ class AWSEventStreamDecoder: index=_index, is_finished=True, finish_reason="stop", + usage=None, ) return GChunk( @@ -875,9 +893,12 @@ class AWSEventStreamDecoder: index=_index, is_finished=is_finished, finish_reason=finish_reason, + usage=None, ) - def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[GChunk]: + def iter_bytes( + self, iterator: Iterator[bytes] + ) -> Iterator[Optional[Union[GChunk, StreamingChatCompletionChunk]]]: """Given an iterator that yields lines, iterate over it & yield every event encountered""" from botocore.eventstream import EventStreamBuffer @@ -898,7 +919,10 @@ class AWSEventStreamDecoder: # Try to parse the accumulated JSON try: _data = json.loads(accumulated_json) - yield self._chunk_parser(chunk_data=_data) + if self.is_messages_api: + yield self._chunk_parser_messages_api(chunk_data=_data) + else: + yield self._chunk_parser(chunk_data=_data) # Reset accumulated_json after successful parsing accumulated_json = "" except json.JSONDecodeError: @@ -909,16 +933,20 @@ class AWSEventStreamDecoder: if accumulated_json: try: _data = json.loads(accumulated_json) - yield self._chunk_parser(chunk_data=_data) - except json.JSONDecodeError: + if self.is_messages_api: + yield self._chunk_parser_messages_api(chunk_data=_data) + else: + yield self._chunk_parser(chunk_data=_data) + except json.JSONDecodeError as e: # Handle or log any unparseable data at the end verbose_logger.error( f"Warning: Unparseable JSON data remained: {accumulated_json}" ) + yield None async def aiter_bytes( self, iterator: AsyncIterator[bytes] - ) -> AsyncIterator[GChunk]: + ) -> AsyncIterator[Optional[Union[GChunk, StreamingChatCompletionChunk]]]: """Given an async iterator that yields lines, iterate over it & yield every event encountered""" from botocore.eventstream import EventStreamBuffer @@ -940,7 +968,10 @@ class AWSEventStreamDecoder: # Try to parse the accumulated JSON try: _data = json.loads(accumulated_json) - yield self._chunk_parser(chunk_data=_data) + if self.is_messages_api: + yield self._chunk_parser_messages_api(chunk_data=_data) + else: + yield self._chunk_parser(chunk_data=_data) # Reset accumulated_json after successful parsing accumulated_json = "" except json.JSONDecodeError: @@ -951,12 +982,16 @@ class AWSEventStreamDecoder: if accumulated_json: try: _data = json.loads(accumulated_json) - yield self._chunk_parser(chunk_data=_data) + if self.is_messages_api: + yield self._chunk_parser_messages_api(chunk_data=_data) + else: + yield self._chunk_parser(chunk_data=_data) except json.JSONDecodeError: # Handle or log any unparseable data at the end verbose_logger.error( f"Warning: Unparseable JSON data remained: {accumulated_json}" ) + yield None def _parse_message_from_event(self, event) -> Optional[str]: response_dict = event.to_response_dict() diff --git a/litellm/main.py b/litellm/main.py index 2cf836890..adc711735 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -119,7 +119,7 @@ from .llms.prompt_templates.factory import ( prompt_factory, stringify_json_tool_call_content, ) -from .llms.sagemaker import SagemakerLLM +from .llms.sagemaker.sagemaker import SagemakerLLM from .llms.text_completion_codestral import CodestralTextCompletion from .llms.text_to_speech.vertex_ai import VertexTextToSpeechAPI from .llms.triton import TritonChatCompletion diff --git a/litellm/tests/test_sagemaker.py b/litellm/tests/test_sagemaker.py index f407191f8..f0b77af4f 100644 --- a/litellm/tests/test_sagemaker.py +++ b/litellm/tests/test_sagemaker.py @@ -120,15 +120,24 @@ async def test_completion_sagemaker_messages_api(sync_mode): @pytest.mark.asyncio() @pytest.mark.parametrize("sync_mode", [False, True]) -async def test_completion_sagemaker_stream(sync_mode): +@pytest.mark.parametrize( + "model", + [ + "sagemaker_chat/huggingface-pytorch-tgi-inference-2024-08-23-15-48-59-245", + "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + ], +) +async def test_completion_sagemaker_stream(sync_mode, model): try: + from litellm.tests.test_streaming import streaming_format_tests + litellm.set_verbose = False print("testing sagemaker") verbose_logger.setLevel(logging.DEBUG) full_text = "" if sync_mode is True: response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model=model, messages=[ {"role": "user", "content": "hi - what is ur name"}, ], @@ -138,14 +147,15 @@ async def test_completion_sagemaker_stream(sync_mode): input_cost_per_second=0.000420, ) - for chunk in response: + for idx, chunk in enumerate(response): print(chunk) + streaming_format_tests(idx=idx, chunk=chunk) full_text += chunk.choices[0].delta.content or "" print("SYNC RESPONSE full text", full_text) else: response = await litellm.acompletion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + model=model, messages=[ {"role": "user", "content": "hi - what is ur name"}, ], @@ -156,10 +166,12 @@ async def test_completion_sagemaker_stream(sync_mode): ) print("streaming response") - + idx = 0 async for chunk in response: print(chunk) + streaming_format_tests(idx=idx, chunk=chunk) full_text += chunk.choices[0].delta.content or "" + idx += 1 print("ASYNC RESPONSE full text", full_text) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 5d2c416f9..8d7520f25 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -29,6 +29,7 @@ from openai.types.beta.thread_create_params import ( from openai.types.beta.threads.message import Message as OpenAIMessage from openai.types.beta.threads.message_content import MessageContent from openai.types.beta.threads.run import Run +from openai.types.chat import ChatCompletionChunk from pydantic import BaseModel, Field from typing_extensions import Dict, Required, TypedDict, override @@ -456,6 +457,13 @@ class ChatCompletionUsageBlock(TypedDict): total_tokens: int +class OpenAIChatCompletionChunk(ChatCompletionChunk): + def __init__(self, **kwargs): + # Set the 'object' kwarg to 'chat.completion.chunk' + kwargs["object"] = "chat.completion.chunk" + super().__init__(**kwargs) + + class Hyperparameters(BaseModel): batch_size: Optional[Union[str, int]] = None # "Number of examples in each batch." learning_rate_multiplier: Optional[Union[str, float]] = ( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 14d5cd1b8..3e5465bdc 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -5,11 +5,16 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Tuple, Union from openai._models import BaseModel as OpenAIObject +from openai.types.completion_usage import CompletionUsage from pydantic import ConfigDict, Field, PrivateAttr from typing_extensions import Callable, Dict, Required, TypedDict, override from ..litellm_core_utils.core_helpers import map_finish_reason -from .llms.openai import ChatCompletionToolCallChunk, ChatCompletionUsageBlock +from .llms.openai import ( + ChatCompletionToolCallChunk, + ChatCompletionUsageBlock, + OpenAIChatCompletionChunk, +) def _generate_id(): # private helper function @@ -85,7 +90,7 @@ class GenericStreamingChunk(TypedDict, total=False): tool_use: Optional[ChatCompletionToolCallChunk] is_finished: Required[bool] finish_reason: Required[str] - usage: Optional[ChatCompletionUsageBlock] + usage: Required[Optional[ChatCompletionUsageBlock]] index: int # use this dict if you want to return any provider specific fields in the response @@ -448,9 +453,6 @@ class Choices(OpenAIObject): setattr(self, key, value) -from openai.types.completion_usage import CompletionUsage - - class Usage(CompletionUsage): def __init__( self, @@ -535,6 +537,17 @@ class StreamingChoices(OpenAIObject): setattr(self, key, value) +class StreamingChatCompletionChunk(OpenAIChatCompletionChunk): + def __init__(self, **kwargs): + + new_choices = [] + for choice in kwargs["choices"]: + new_choice = StreamingChoices(**choice).model_dump() + new_choices.append(new_choice) + kwargs["choices"] = new_choices + super().__init__(**kwargs) + + class ModelResponse(OpenAIObject): id: str """A unique identifier for the completion.""" @@ -1231,3 +1244,20 @@ class StandardLoggingPayload(TypedDict): response: Optional[Union[str, list, dict]] model_parameters: dict hidden_params: StandardLoggingHiddenParams + + +from typing import AsyncIterator, Iterator + + +class CustomStreamingDecoder: + async def aiter_bytes( + self, iterator: AsyncIterator[bytes] + ) -> AsyncIterator[ + Optional[Union[GenericStreamingChunk, StreamingChatCompletionChunk]] + ]: + raise NotImplementedError + + def iter_bytes( + self, iterator: Iterator[bytes] + ) -> Iterator[Optional[Union[GenericStreamingChunk, StreamingChatCompletionChunk]]]: + raise NotImplementedError From 0a15d3b3c3c8f4f4fc31a0f3f01b1dc38bde93ae Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 26 Aug 2024 15:42:55 -0700 Subject: [PATCH 20/48] fix(utils.py): fix message replace --- litellm/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 68eac7c3a..81ba95087 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6649,9 +6649,14 @@ def exception_type( else: message = str(original_exception) - if message is not None and isinstance(message, str): + if message is not None and isinstance( + message, str + ): # done to prevent user-confusion. Relevant issue - https://github.com/BerriAI/litellm/issues/1414 message = message.replace("OPENAI", custom_llm_provider.upper()) - message = message.replace("OpenAI", custom_llm_provider) + message = message.replace( + "openai.OpenAIError", + "{}.{}Error".format(custom_llm_provider, custom_llm_provider), + ) if custom_llm_provider == "openai": exception_provider = "OpenAI" + "Exception" else: From 4f8026f44d5e5947fd0bfd8d6a36e1ca8b6a2365 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 16:33:04 -0700 Subject: [PATCH 21/48] fix refactor cohere --- litellm/llms/cohere.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 litellm/llms/cohere.py diff --git a/litellm/llms/cohere.py b/litellm/llms/cohere.py deleted file mode 100644 index e69de29bb..000000000 From fb150f7ce5032946af4aef739180cc3c5d172c98 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 16:52:19 -0700 Subject: [PATCH 22/48] update schema --- litellm/proxy/schema.prisma | 3 +++ schema.prisma | 3 +++ 2 files changed, 6 insertions(+) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index cf61635a0..55bfdb2e0 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -149,6 +149,9 @@ model LiteLLM_VerificationToken { model_max_budget Json @default("{}") budget_id String? litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) + key_state String? // can be "active", "inactive" + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") } model LiteLLM_EndUserTable { diff --git a/schema.prisma b/schema.prisma index 8f4125104..01fe468d0 100644 --- a/schema.prisma +++ b/schema.prisma @@ -149,6 +149,9 @@ model LiteLLM_VerificationToken { model_max_budget Json @default("{}") budget_id String? litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) + key_state String? // can be "active", "inactive" + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") } model LiteLLM_EndUserTable { From cbef0c0a0dd8abcd0f86203df3f494cfae5dbfc8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 16:52:33 -0700 Subject: [PATCH 23/48] add key_state created at to token --- litellm/proxy/_types.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d660e576d..260f0431d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1299,8 +1299,10 @@ class LiteLLM_VerificationToken(LiteLLMBase): model_max_budget: Dict = {} soft_budget_cooldown: bool = False litellm_budget_table: Optional[dict] = None - org_id: Optional[str] = None # org id for a given key + key_state: Optional[str] = None # can be "active", "inactive" + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None model_config = ConfigDict(protected_namespaces=()) From 5745f3d6cc8e115a38297222fd498c6b8689432d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 17:27:06 -0700 Subject: [PATCH 24/48] fix schema --- litellm/proxy/_types.py | 1 - litellm/proxy/schema.prisma | 1 - schema.prisma | 1 - 3 files changed, 3 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 260f0431d..554c9e4e4 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1300,7 +1300,6 @@ class LiteLLM_VerificationToken(LiteLLMBase): soft_budget_cooldown: bool = False litellm_budget_table: Optional[dict] = None org_id: Optional[str] = None # org id for a given key - key_state: Optional[str] = None # can be "active", "inactive" created_at: Optional[datetime] = None updated_at: Optional[datetime] = None diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 55bfdb2e0..1af0c0a34 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -149,7 +149,6 @@ model LiteLLM_VerificationToken { model_max_budget Json @default("{}") budget_id String? litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) - key_state String? // can be "active", "inactive" created_at DateTime @default(now()) @map("created_at") updated_at DateTime @default(now()) @updatedAt @map("updated_at") } diff --git a/schema.prisma b/schema.prisma index 01fe468d0..86ec201f7 100644 --- a/schema.prisma +++ b/schema.prisma @@ -149,7 +149,6 @@ model LiteLLM_VerificationToken { model_max_budget Json @default("{}") budget_id String? litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id]) - key_state String? // can be "active", "inactive" created_at DateTime @default(now()) @map("created_at") updated_at DateTime @default(now()) @updatedAt @map("updated_at") } From 7230ee1f55b934d54bb0fa5b7fc3f275e451059c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 17:59:44 -0700 Subject: [PATCH 25/48] add regenerate_key_fn --- .../key_management_endpoints.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2e16b533c..ebe1d9db3 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -966,3 +966,82 @@ async def delete_verification_token(tokens: List, user_id: Optional[str] = None) verbose_proxy_logger.debug(traceback.format_exc()) raise e return deleted_tokens + + +@router.post( + "/key/{key:path}/regenerate", + tags=["key management"], + dependencies=[Depends(user_api_key_auth)], +) +@management_endpoint_wrapper +async def regenerate_key_fn( + key: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +) -> GenerateKeyResponse: + from litellm.proxy.proxy_server import ( + hash_token, + premium_user, + prisma_client, + user_api_key_cache, + ) + + """ + Endpoint for regenerating a key + """ + + # Check if key exists, raise exception if key is not in the DB + + ### 1. Create New copy that is duplicate of existing key + ###################################################################### + + # create duplicate of existing key + # set token = new token generated + # insert new token in DB + + # create hash of token + if prisma_client is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": "DB not connected. prisma_client is None"}, + ) + hashed_api_key = hash_token(key) + + _key_in_db = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_api_key}, + ) + if _key_in_db is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": f"Key {key} not found."}, + ) + + verbose_proxy_logger.debug("key_in_db: %s", _key_in_db) + + new_token = f"sk-{secrets.token_urlsafe(16)}" + new_token_hash = hash_token(new_token) + + # update new token in DB + updated_token = await prisma_client.db.litellm_verificationtoken.update( + where={"token": hashed_api_key}, data={"token": new_token_hash} + ) + updated_token_dict = {} + if updated_token is not None: + updated_token_dict = dict(updated_token) + + updated_token_dict["token"] = new_token + + ### 3. remove existing key entry from cache + ###################################################################### + if key: + user_api_key_cache.delete_cache(key) + + if hashed_api_key: + user_api_key_cache.delete_cache(hashed_api_key) + + return GenerateKeyResponse( + **updated_token_dict, + ) From 6b8bb69af399caefc9bdb980d163d2dc75d8f800 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:00:51 -0700 Subject: [PATCH 26/48] test test_regenerate_api_key --- litellm/tests/test_key_generate_prisma.py | 101 ++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/litellm/tests/test_key_generate_prisma.py b/litellm/tests/test_key_generate_prisma.py index 8eedd639f..e41a67c08 100644 --- a/litellm/tests/test_key_generate_prisma.py +++ b/litellm/tests/test_key_generate_prisma.py @@ -56,6 +56,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_fn, generate_key_helper_fn, info_key_fn, + regenerate_key_fn, update_key_fn, ) from litellm.proxy.management_endpoints.team_endpoints import ( @@ -2935,3 +2936,103 @@ async def test_team_access_groups(prisma_client): "not allowed to call model" in e.message and "Allowed team models" in e.message ) + + +################ Unit Tests for testing regeneration of keys ########### +@pytest.mark.asyncio() +async def test_regenerate_api_key(prisma_client): + litellm.set_verbose = True + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + await litellm.proxy.proxy_server.prisma_client.connect() + import uuid + + # generate new key + key_alias = f"test_alias_regenerate_key-{uuid.uuid4()}" + spend = 100 + max_budget = 400 + models = ["fake-openai-endpoint"] + new_key = await generate_key_fn( + data=GenerateKeyRequest( + key_alias=key_alias, spend=spend, max_budget=max_budget, models=models + ), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + ) + + generated_key = new_key.key + print(generated_key) + + # assert the new key works as expected + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + + async def return_body(): + return_string = f'{{"model": "fake-openai-endpoint"}}' + # return string as bytes + return return_string.encode() + + request.body = return_body + result = await user_api_key_auth(request=request, api_key=f"Bearer {generated_key}") + print(result) + + # regenerate the key + new_key = await regenerate_key_fn( + key=generated_key, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + ) + print("response from regenerate_key_fn", new_key) + + # assert the new key works as expected + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + + async def return_body_2(): + return_string = f'{{"model": "fake-openai-endpoint"}}' + # return string as bytes + return return_string.encode() + + request.body = return_body_2 + result = await user_api_key_auth(request=request, api_key=f"Bearer {new_key.key}") + print(result) + + # assert the old key stops working + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + + async def return_body_3(): + return_string = f'{{"model": "fake-openai-endpoint"}}' + # return string as bytes + return return_string.encode() + + request.body = return_body_3 + try: + result = await user_api_key_auth( + request=request, api_key=f"Bearer {generated_key}" + ) + print(result) + pytest.fail(f"This should have failed!. the key has been regenerated") + except Exception as e: + print("got expected exception", e) + assert "Invalid proxy server token passed" in e.message + + # Check that the regenerated key has the same spend, max_budget, models and key_alias + assert new_key.spend == spend, f"Expected spend {spend} but got {new_key.spend}" + assert ( + new_key.max_budget == max_budget + ), f"Expected max_budget {max_budget} but got {new_key.max_budget}" + assert ( + new_key.key_alias == key_alias + ), f"Expected key_alias {key_alias} but got {new_key.key_alias}" + assert ( + new_key.models == models + ), f"Expected models {models} but got {new_key.models}" + + pass From 2615edc468042333fbe9965e88fb06113fd77d03 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:15:52 -0700 Subject: [PATCH 27/48] allow using hashed api keys on regen key --- .../proxy/management_endpoints/key_management_endpoints.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index ebe1d9db3..aef2fb393 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1008,7 +1008,11 @@ async def regenerate_key_fn( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": "DB not connected. prisma_client is None"}, ) - hashed_api_key = hash_token(key) + + if "sk" not in key: + hashed_api_key = key + else: + hashed_api_key = hash_token(key) _key_in_db = await prisma_client.db.litellm_verificationtoken.find_unique( where={"token": hashed_api_key}, From 40c018272c6735202797662fc563bc83d5dd0acf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:28:26 -0700 Subject: [PATCH 28/48] working regenerate key flow --- .../src/components/networking.tsx | 31 +++++++ .../src/components/view_key_table.tsx | 83 ++++++++++++++++++- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index f55076478..a6bd5d32c 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -770,6 +770,37 @@ export const claimOnboardingToken = async ( throw error; } }; + +export const regenerateKeyCall = async (accessToken: string, keyToRegenerate: string) => { + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/key/${keyToRegenerate}/regenerate` + : `/key/${keyToRegenerate}/regenerate`; + + const response = await fetch(url, { + method: "POST", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({}), + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + console.log("Regenerate key Response:", data); + return data; + } catch (error) { + console.error("Failed to regenerate key:", error); + throw error; + } +}; + let ModelListerrorShown = false; let errorTimer: NodeJS.Timeout | null = null; diff --git a/ui/litellm-dashboard/src/components/view_key_table.tsx b/ui/litellm-dashboard/src/components/view_key_table.tsx index 44a1f6ece..752aad37f 100644 --- a/ui/litellm-dashboard/src/components/view_key_table.tsx +++ b/ui/litellm-dashboard/src/components/view_key_table.tsx @@ -1,12 +1,14 @@ "use client"; import React, { useEffect, useState } from "react"; import { keyDeleteCall, modelAvailableCall } from "./networking"; -import { InformationCircleIcon, StatusOnlineIcon, TrashIcon, PencilAltIcon } from "@heroicons/react/outline"; -import { keySpendLogsCall, PredictedSpendLogsCall, keyUpdateCall, modelInfoCall } from "./networking"; +import { InformationCircleIcon, StatusOnlineIcon, TrashIcon, PencilAltIcon, RefreshIcon } from "@heroicons/react/outline"; +import { keySpendLogsCall, PredictedSpendLogsCall, keyUpdateCall, modelInfoCall, regenerateKeyCall } from "./networking"; import { Badge, Card, Table, + Grid, + Col, Button, TableBody, TableCell, @@ -33,6 +35,8 @@ import { Select, } from "antd"; +import { CopyToClipboard } from "react-copy-to-clipboard"; + const { Option } = Select; const isLocal = process.env.NODE_ENV === "development"; const proxyBaseUrl = isLocal ? "http://localhost:4000" : null; @@ -109,6 +113,8 @@ const ViewKeyTable: React.FC = ({ const [userModels, setUserModels] = useState([]); const initialKnownTeamIDs: Set = new Set(); const [modelLimitModalVisible, setModelLimitModalVisible] = useState(false); + const [regenerateDialogVisible, setRegenerateDialogVisible] = useState(false); + const [regeneratedKey, setRegeneratedKey] = useState(null); const [knownTeamIDs, setKnownTeamIDs] = useState(initialKnownTeamIDs); @@ -612,6 +618,18 @@ const ViewKeyTable: React.FC = ({ setKeyToDelete(null); }; + const handleRegenerateKey = async () => { + try { + const response = await regenerateKeyCall(accessToken, selectedToken.token); + setRegeneratedKey(response.key); + setRegenerateDialogVisible(false); + message.success("API Key regenerated successfully"); + } catch (error) { + console.error("Error regenerating key:", error); + message.error("Failed to regenerate API Key"); + } + }; + if (data == null) { return; } @@ -768,6 +786,7 @@ const ViewKeyTable: React.FC = ({ size="sm" /> + = ({ size="sm" onClick={() => handleEditClick(item)} /> + { + setSelectedToken(item); + setRegenerateDialogVisible(true); + }} + icon={RefreshIcon} + size="sm" + /> handleDelete(item)} icon={TrashIcon} @@ -942,6 +969,58 @@ const ViewKeyTable: React.FC = ({ accessToken={accessToken} /> )} + + {/* Regenerate Key Confirmation Dialog */} + setRegenerateDialogVisible(false)} + > +

Are you sure you want to regenerate this key?

+

Key Alias:

+
{selectedToken?.key_alias || 'No alias set'}
+
+ + {/* Regenerated Key Display Modal */} + {regeneratedKey && ( + setRegeneratedKey(null)} + onCancel={() => setRegeneratedKey(null)} + footer={null} + > + + Save your New Key + +

+ Please save this new secret key somewhere safe and accessible. For + security reasons, you will not be able to view it again through + your LiteLLM account. If you lose this secret key, you will need to + generate a new one. +

+ + + New API Key: +
+
+                {regeneratedKey}
+              
+
+ message.success("API Key copied to clipboard")}> + + + +
+
+ )} ); }; From 3527e47b59fe169a261028aec8fd2e8e3b348f6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:33:18 -0700 Subject: [PATCH 29/48] ui regenerate an api key --- .../src/components/view_key_table.tsx | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_key_table.tsx b/ui/litellm-dashboard/src/components/view_key_table.tsx index 752aad37f..620c8ca1c 100644 --- a/ui/litellm-dashboard/src/components/view_key_table.tsx +++ b/ui/litellm-dashboard/src/components/view_key_table.tsx @@ -620,6 +620,10 @@ const ViewKeyTable: React.FC = ({ const handleRegenerateKey = async () => { try { + if (selectedToken == null) { + message.error("Please select a key to regenerate"); + return; + } const response = await regenerateKeyCall(accessToken, selectedToken.token); setRegeneratedKey(response.key); setRegenerateDialogVisible(false); @@ -974,8 +978,15 @@ const ViewKeyTable: React.FC = ({ setRegenerateDialogVisible(false)} + footer={[ + , + + ]} >

Are you sure you want to regenerate this key?

Key Alias:

@@ -986,9 +997,12 @@ const ViewKeyTable: React.FC = ({ {regeneratedKey && ( setRegeneratedKey(null)} onCancel={() => setRegeneratedKey(null)} - footer={null} + footer={[ + + ]} > Save your New Key From 4dc2eea58d0c11627792fef20c6fcb50a357551a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:40:51 -0700 Subject: [PATCH 30/48] update key name when regenerating a key --- .../proxy/management_endpoints/key_management_endpoints.py | 7 ++++++- litellm/tests/test_key_generate_prisma.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index aef2fb393..4688445fe 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1027,10 +1027,15 @@ async def regenerate_key_fn( new_token = f"sk-{secrets.token_urlsafe(16)}" new_token_hash = hash_token(new_token) + new_token_key_name = f"sk-...{new_token[-4:]}" # update new token in DB updated_token = await prisma_client.db.litellm_verificationtoken.update( - where={"token": hashed_api_key}, data={"token": new_token_hash} + where={"token": hashed_api_key}, + data={ + "token": new_token_hash, + "key_name": new_token_key_name, + }, ) updated_token_dict = {} if updated_token is not None: diff --git a/litellm/tests/test_key_generate_prisma.py b/litellm/tests/test_key_generate_prisma.py index e41a67c08..49a4c95c7 100644 --- a/litellm/tests/test_key_generate_prisma.py +++ b/litellm/tests/test_key_generate_prisma.py @@ -3035,4 +3035,6 @@ async def test_regenerate_api_key(prisma_client): new_key.models == models ), f"Expected models {models} but got {new_key.models}" + assert new_key.key_name == f"sk-...{new_key.key[-4:]}" + pass From 5c2c316919743b0bb2219d059b38256dea69d0f3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:41:20 -0700 Subject: [PATCH 31/48] update ui regen key --- ui/litellm-dashboard/src/components/view_key_table.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_key_table.tsx b/ui/litellm-dashboard/src/components/view_key_table.tsx index 620c8ca1c..41d930fc3 100644 --- a/ui/litellm-dashboard/src/components/view_key_table.tsx +++ b/ui/litellm-dashboard/src/components/view_key_table.tsx @@ -1005,10 +1005,10 @@ const ViewKeyTable: React.FC = ({ ]} > - Save your New Key + Regenerated Key

- Please save this new secret key somewhere safe and accessible. For + Please replace your old key with the new key generated.For security reasons, you will not be able to view it again through your LiteLLM account. If you lose this secret key, you will need to generate a new one. From 83813af0a2ec6e1dc1314cb44e7dff1b58b11e7b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:44:46 -0700 Subject: [PATCH 32/48] regenerate key --- .../src/components/view_key_table.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ui/litellm-dashboard/src/components/view_key_table.tsx b/ui/litellm-dashboard/src/components/view_key_table.tsx index 41d930fc3..d9e73b109 100644 --- a/ui/litellm-dashboard/src/components/view_key_table.tsx +++ b/ui/litellm-dashboard/src/components/view_key_table.tsx @@ -626,6 +626,17 @@ const ViewKeyTable: React.FC = ({ } const response = await regenerateKeyCall(accessToken, selectedToken.token); setRegeneratedKey(response.key); + + // Update the data state with the new key_name + if (data) { + const updatedData = data.map(item => + item.token === selectedToken.token + ? { ...item, key_name: response.key_name } + : item + ); + setData(updatedData); + } + setRegenerateDialogVisible(false); message.success("API Key regenerated successfully"); } catch (error) { From b0ae0101f49d69eb8e98ebed4e5c7f2ba30dadfc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:53:28 -0700 Subject: [PATCH 33/48] make regenerating api keys enterprise --- ui/litellm-dashboard/src/app/page.tsx | 2 ++ .../src/components/user_dashboard.tsx | 3 ++ .../src/components/view_key_table.tsx | 36 +++++++++++++++---- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 02ef8ebe0..b35d99cad 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -141,6 +141,7 @@ const CreateKeyPage = () => { { >; setProxySettings: React.Dispatch>; proxySettings: any; + premiumUser: boolean; } type TeamInterface = { @@ -68,6 +69,7 @@ const UserDashboard: React.FC = ({ setKeys, setProxySettings, proxySettings, + premiumUser, }) => { const [userSpendData, setUserSpendData] = useState( null @@ -328,6 +330,7 @@ const UserDashboard: React.FC = ({ selectedTeam={selectedTeam ? selectedTeam : null} data={keys} setData={setKeys} + premiumUser={premiumUser} teams={teams} /> >; teams: any[] | null; + premiumUser: boolean; } interface ItemData { @@ -96,7 +97,8 @@ const ViewKeyTable: React.FC = ({ selectedTeam, data, setData, - teams + teams, + premiumUser }) => { const [isButtonClicked, setIsButtonClicked] = useState(false); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); @@ -619,6 +621,11 @@ const ViewKeyTable: React.FC = ({ }; const handleRegenerateKey = async () => { + if (!premiumUser) { + message.error("Regenerate API Key is an Enterprise feature. Please upgrade to use this feature."); + return; + } + try { if (selectedToken == null) { message.error("Please select a key to regenerate"); @@ -994,14 +1001,31 @@ const ViewKeyTable: React.FC = ({ , - ]} > -

Are you sure you want to regenerate this key?

-

Key Alias:

-
{selectedToken?.key_alias || 'No alias set'}
+ {premiumUser ? ( + <> +

Are you sure you want to regenerate this key?

+

Key Alias:

+
{selectedToken?.key_alias || 'No alias set'}
+ + ) : ( +
+

Upgrade to use this feature

+ +
+ )}
{/* Regenerated Key Display Modal */} From 75cbbea0719bf646a9c042d89af88f224b08fdbe Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:54:50 -0700 Subject: [PATCH 34/48] enforce regenerating keys in enterprise tier --- .../proxy/management_endpoints/key_management_endpoints.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 4688445fe..9bb07cfee 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -993,6 +993,11 @@ async def regenerate_key_fn( Endpoint for regenerating a key """ + if premium_user is not True: + raise ValueError( + f"Regenerating Virtual Keys is an Enterprise feature, {CommonProxyErrors.not_premium_user.value}" + ) + # Check if key exists, raise exception if key is not in the DB ### 1. Create New copy that is duplicate of existing key From a043676dc48cf0ec75908b99df240c641c7817a9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 26 Aug 2024 18:59:29 -0700 Subject: [PATCH 35/48] fix regen api key flow --- .../src/components/view_key_table.tsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/view_key_table.tsx b/ui/litellm-dashboard/src/components/view_key_table.tsx index e0ac43521..019a11179 100644 --- a/ui/litellm-dashboard/src/components/view_key_table.tsx +++ b/ui/litellm-dashboard/src/components/view_key_table.tsx @@ -1043,13 +1043,26 @@ const ViewKeyTable: React.FC = ({ Regenerated Key

- Please replace your old key with the new key generated.For + Please replace your old key with the new key generated. For security reasons, you will not be able to view it again through your LiteLLM account. If you lose this secret key, you will need to generate a new one.

+ Key Alias: +
+
+                {selectedToken?.key_alias || 'No alias set'}
+              
+
New API Key:
Date: Mon, 26 Aug 2024 19:01:35 -0700 Subject: [PATCH 36/48] ui new build --- litellm/model_prices_and_context_window_backup.json | 12 ++++++++++++ litellm/proxy/_experimental/out/404.html | 1 + .../_buildManifest.js | 0 .../_ssgManifest.js | 0 .../out/_next/static/chunks/131-73d0a4f8e09896fe.js | 8 ++++++++ .../out/_next/static/chunks/131-cb6bfe24e23e121b.js | 8 -------- ...5-8e4b96f972af8eaf.js => 605-35a95945041f7699.js} | 4 ++-- .../out/_next/static/chunks/777-50d836152fad178b.js | 1 - .../out/_next/static/chunks/777-5360b5460eba0779.js | 1 + ...-79eee78ed9fccf89.js => page-baad96761e038837.js} | 0 ...-8be9c2a4a5c886c5.js => page-0034957a9fa387e0.js} | 0 .../_next/static/chunks/app/page-01641b817a14ea88.js | 1 + .../_next/static/chunks/app/page-b77076dbc8208d12.js | 1 - litellm/proxy/_experimental/out/index.html | 2 +- litellm/proxy/_experimental/out/index.txt | 4 ++-- litellm/proxy/_experimental/out/model_hub.html | 1 + litellm/proxy/_experimental/out/model_hub.txt | 4 ++-- litellm/proxy/_experimental/out/onboarding.html | 1 + litellm/proxy/_experimental/out/onboarding.txt | 4 ++-- ui/litellm-dashboard/out/404.html | 2 +- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 .../out/_next/static/chunks/131-73d0a4f8e09896fe.js | 8 ++++++++ .../out/_next/static/chunks/131-cb6bfe24e23e121b.js | 8 -------- ...5-8e4b96f972af8eaf.js => 605-35a95945041f7699.js} | 4 ++-- .../out/_next/static/chunks/777-50d836152fad178b.js | 1 - .../out/_next/static/chunks/777-5360b5460eba0779.js | 1 + ...-79eee78ed9fccf89.js => page-baad96761e038837.js} | 0 ...-8be9c2a4a5c886c5.js => page-0034957a9fa387e0.js} | 0 .../_next/static/chunks/app/page-01641b817a14ea88.js | 1 + .../_next/static/chunks/app/page-b77076dbc8208d12.js | 1 - ui/litellm-dashboard/out/index.html | 2 +- ui/litellm-dashboard/out/index.txt | 4 ++-- ui/litellm-dashboard/out/model_hub.html | 2 +- ui/litellm-dashboard/out/model_hub.txt | 4 ++-- ui/litellm-dashboard/out/onboarding.html | 2 +- ui/litellm-dashboard/out/onboarding.txt | 4 ++-- 37 files changed, 56 insertions(+), 41 deletions(-) create mode 100644 litellm/proxy/_experimental/out/404.html rename litellm/proxy/_experimental/out/_next/static/{cjLC-FNUY9ME2ZrO3jtsn => LO0Sm6uVF0pa4RdHSL0dN}/_buildManifest.js (100%) rename litellm/proxy/_experimental/out/_next/static/{cjLC-FNUY9ME2ZrO3jtsn => LO0Sm6uVF0pa4RdHSL0dN}/_ssgManifest.js (100%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/131-73d0a4f8e09896fe.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/131-cb6bfe24e23e121b.js rename litellm/proxy/_experimental/out/_next/static/chunks/{605-8e4b96f972af8eaf.js => 605-35a95945041f7699.js} (53%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/777-50d836152fad178b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/777-5360b5460eba0779.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/{page-79eee78ed9fccf89.js => page-baad96761e038837.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/{page-8be9c2a4a5c886c5.js => page-0034957a9fa387e0.js} (100%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-01641b817a14ea88.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-b77076dbc8208d12.js create mode 100644 litellm/proxy/_experimental/out/model_hub.html create mode 100644 litellm/proxy/_experimental/out/onboarding.html rename ui/litellm-dashboard/out/_next/static/{cjLC-FNUY9ME2ZrO3jtsn => LO0Sm6uVF0pa4RdHSL0dN}/_buildManifest.js (100%) rename ui/litellm-dashboard/out/_next/static/{cjLC-FNUY9ME2ZrO3jtsn => LO0Sm6uVF0pa4RdHSL0dN}/_ssgManifest.js (100%) create mode 100644 ui/litellm-dashboard/out/_next/static/chunks/131-73d0a4f8e09896fe.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/131-cb6bfe24e23e121b.js rename ui/litellm-dashboard/out/_next/static/chunks/{605-8e4b96f972af8eaf.js => 605-35a95945041f7699.js} (53%) delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/777-50d836152fad178b.js create mode 100644 ui/litellm-dashboard/out/_next/static/chunks/777-5360b5460eba0779.js rename ui/litellm-dashboard/out/_next/static/chunks/app/model_hub/{page-79eee78ed9fccf89.js => page-baad96761e038837.js} (100%) rename ui/litellm-dashboard/out/_next/static/chunks/app/onboarding/{page-8be9c2a4a5c886c5.js => page-0034957a9fa387e0.js} (100%) create mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/page-01641b817a14ea88.js delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/page-b77076dbc8208d12.js diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0f483b083..4f5fffc52 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2189,6 +2189,18 @@ "mode": "image_generation", "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/imagen-3.0-generate-001": { + "cost_per_image": 0.04, + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + }, + "vertex_ai/imagen-3.0-fast-generate-001": { + "cost_per_image": 0.02, + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + }, "text-embedding-004": { "max_tokens": 3072, "max_input_tokens": 3072, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html new file mode 100644 index 000000000..1939ceec4 --- /dev/null +++ b/litellm/proxy/_experimental/out/404.html @@ -0,0 +1 @@ +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/cjLC-FNUY9ME2ZrO3jtsn/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/LO0Sm6uVF0pa4RdHSL0dN/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/cjLC-FNUY9ME2ZrO3jtsn/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/LO0Sm6uVF0pa4RdHSL0dN/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/cjLC-FNUY9ME2ZrO3jtsn/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/LO0Sm6uVF0pa4RdHSL0dN/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/cjLC-FNUY9ME2ZrO3jtsn/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/LO0Sm6uVF0pa4RdHSL0dN/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/131-73d0a4f8e09896fe.js b/litellm/proxy/_experimental/out/_next/static/chunks/131-73d0a4f8e09896fe.js new file mode 100644 index 000000000..f012416c9 --- /dev/null +++ b/litellm/proxy/_experimental/out/_next/static/chunks/131-73d0a4f8e09896fe.js @@ -0,0 +1,8 @@ +"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[131],{84174:function(e,t,n){n.d(t,{Z:function(){return s}});var a=n(14749),r=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M832 64H296c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h496v688c0 4.4 3.6 8 8 8h56c4.4 0 8-3.6 8-8V96c0-17.7-14.3-32-32-32zM704 192H192c-17.7 0-32 14.3-32 32v530.7c0 8.5 3.4 16.6 9.4 22.6l173.3 173.3c2.2 2.2 4.7 4 7.4 5.5v1.9h4.2c3.5 1.3 7.2 2 11 2H704c17.7 0 32-14.3 32-32V224c0-17.7-14.3-32-32-32zM350 856.2L263.9 770H350v86.2zM664 888H414V746c0-22.1-17.9-40-40-40H232V264h432v624z"}}]},name:"copy",theme:"outlined"},o=n(60688),s=r.forwardRef(function(e,t){return r.createElement(o.Z,(0,a.Z)({},e,{ref:t,icon:i}))})},50459:function(e,t,n){n.d(t,{Z:function(){return s}});var a=n(14749),r=n(2265),i={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M765.7 486.8L314.9 134.7A7.97 7.97 0 00302 141v77.3c0 4.9 2.3 9.6 6.1 12.6l360 281.1-360 281.1c-3.9 3-6.1 7.7-6.1 12.6V883c0 6.7 7.7 10.4 12.9 6.3l450.8-352.1a31.96 31.96 0 000-50.4z"}}]},name:"right",theme:"outlined"},o=n(60688),s=r.forwardRef(function(e,t){return r.createElement(o.Z,(0,a.Z)({},e,{ref:t,icon:i}))})},92836:function(e,t,n){n.d(t,{Z:function(){return p}});var a=n(69703),r=n(80991),i=n(2898),o=n(99250),s=n(65492),l=n(2265),c=n(41608),d=n(50027);n(18174),n(21871),n(41213);let u=(0,s.fn)("Tab"),p=l.forwardRef((e,t)=>{let{icon:n,className:p,children:g}=e,m=(0,a._T)(e,["icon","className","children"]),b=(0,l.useContext)(c.O),f=(0,l.useContext)(d.Z);return l.createElement(r.O,Object.assign({ref:t,className:(0,o.q)(u("root"),"flex whitespace-nowrap truncate max-w-xs outline-none focus:ring-0 text-tremor-default transition duration-100",f?(0,s.bM)(f,i.K.text).selectTextColor:"solid"===b?"ui-selected:text-tremor-content-emphasis dark:ui-selected:text-dark-tremor-content-emphasis":"ui-selected:text-tremor-brand dark:ui-selected:text-dark-tremor-brand",function(e,t){switch(e){case"line":return(0,o.q)("ui-selected:border-b-2 hover:border-b-2 border-transparent transition duration-100 -mb-px px-2 py-2","hover:border-tremor-content hover:text-tremor-content-emphasis text-tremor-content","dark:hover:border-dark-tremor-content-emphasis dark:hover:text-dark-tremor-content-emphasis dark:text-dark-tremor-content",t?(0,s.bM)(t,i.K.border).selectBorderColor:"ui-selected:border-tremor-brand dark:ui-selected:border-dark-tremor-brand");case"solid":return(0,o.q)("border-transparent border rounded-tremor-small px-2.5 py-1","ui-selected:border-tremor-border ui-selected:bg-tremor-background ui-selected:shadow-tremor-input hover:text-tremor-content-emphasis ui-selected:text-tremor-brand","dark:ui-selected:border-dark-tremor-border dark:ui-selected:bg-dark-tremor-background dark:ui-selected:shadow-dark-tremor-input dark:hover:text-dark-tremor-content-emphasis dark:ui-selected:text-dark-tremor-brand",t?(0,s.bM)(t,i.K.text).selectTextColor:"text-tremor-content dark:text-dark-tremor-content")}}(b,f),p)},m),n?l.createElement(n,{className:(0,o.q)(u("icon"),"flex-none h-5 w-5",g?"mr-2":"")}):null,g?l.createElement("span",null,g):null)});p.displayName="Tab"},26734:function(e,t,n){n.d(t,{Z:function(){return c}});var a=n(69703),r=n(80991),i=n(99250),o=n(65492),s=n(2265);let l=(0,o.fn)("TabGroup"),c=s.forwardRef((e,t)=>{let{defaultIndex:n,index:o,onIndexChange:c,children:d,className:u}=e,p=(0,a._T)(e,["defaultIndex","index","onIndexChange","children","className"]);return s.createElement(r.O.Group,Object.assign({as:"div",ref:t,defaultIndex:n,selectedIndex:o,onChange:c,className:(0,i.q)(l("root"),"w-full",u)},p),d)});c.displayName="TabGroup"},41608:function(e,t,n){n.d(t,{O:function(){return c},Z:function(){return u}});var a=n(69703),r=n(2265),i=n(50027);n(18174),n(21871),n(41213);var o=n(80991),s=n(99250);let l=(0,n(65492).fn)("TabList"),c=(0,r.createContext)("line"),d={line:(0,s.q)("flex border-b space-x-4","border-tremor-border","dark:border-dark-tremor-border"),solid:(0,s.q)("inline-flex p-0.5 rounded-tremor-default space-x-1.5","bg-tremor-background-subtle","dark:bg-dark-tremor-background-subtle")},u=r.forwardRef((e,t)=>{let{color:n,variant:u="line",children:p,className:g}=e,m=(0,a._T)(e,["color","variant","children","className"]);return r.createElement(o.O.List,Object.assign({ref:t,className:(0,s.q)(l("root"),"justify-start overflow-x-clip",d[u],g)},m),r.createElement(c.Provider,{value:u},r.createElement(i.Z.Provider,{value:n},p)))});u.displayName="TabList"},32126:function(e,t,n){n.d(t,{Z:function(){return d}});var a=n(69703);n(50027);var r=n(18174);n(21871);var i=n(41213),o=n(99250),s=n(65492),l=n(2265);let c=(0,s.fn)("TabPanel"),d=l.forwardRef((e,t)=>{let{children:n,className:s}=e,d=(0,a._T)(e,["children","className"]),{selectedValue:u}=(0,l.useContext)(i.Z),p=u===(0,l.useContext)(r.Z);return l.createElement("div",Object.assign({ref:t,className:(0,o.q)(c("root"),"w-full mt-2",p?"":"hidden",s),"aria-selected":p?"true":"false"},d),n)});d.displayName="TabPanel"},23682:function(e,t,n){n.d(t,{Z:function(){return u}});var a=n(69703),r=n(80991);n(50027);var i=n(18174);n(21871);var o=n(41213),s=n(99250),l=n(65492),c=n(2265);let d=(0,l.fn)("TabPanels"),u=c.forwardRef((e,t)=>{let{children:n,className:l}=e,u=(0,a._T)(e,["children","className"]);return c.createElement(r.O.Panels,Object.assign({as:"div",ref:t,className:(0,s.q)(d("root"),"w-full",l)},u),e=>{let{selectedIndex:t}=e;return c.createElement(o.Z.Provider,{value:{selectedValue:t}},c.Children.map(n,(e,t)=>c.createElement(i.Z.Provider,{value:t},e)))})});u.displayName="TabPanels"},50027:function(e,t,n){n.d(t,{Z:function(){return i}});var a=n(2265),r=n(54942);n(99250);let i=(0,a.createContext)(r.fr.Blue)},18174:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(2265).createContext)(0)},21871:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(2265).createContext)(void 0)},41213:function(e,t,n){n.d(t,{Z:function(){return a}});let a=(0,n(2265).createContext)({selectedValue:void 0,handleValueChange:void 0})},21467:function(e,t,n){n.d(t,{i:function(){return s}});var a=n(2265),r=n(44329),i=n(54165),o=n(57499);function s(e){return t=>a.createElement(i.ZP,{theme:{token:{motion:!1,zIndexPopupBase:0}}},a.createElement(e,Object.assign({},t)))}t.Z=(e,t,n,i)=>s(s=>{let{prefixCls:l,style:c}=s,d=a.useRef(null),[u,p]=a.useState(0),[g,m]=a.useState(0),[b,f]=(0,r.Z)(!1,{value:s.open}),{getPrefixCls:E}=a.useContext(o.E_),h=E(t||"select",l);a.useEffect(()=>{if(f(!0),"undefined"!=typeof ResizeObserver){let e=new ResizeObserver(e=>{let t=e[0].target;p(t.offsetHeight+8),m(t.offsetWidth)}),t=setInterval(()=>{var a;let r=n?".".concat(n(h)):".".concat(h,"-dropdown"),i=null===(a=d.current)||void 0===a?void 0:a.querySelector(r);i&&(clearInterval(t),e.observe(i))},10);return()=>{clearInterval(t),e.disconnect()}}},[]);let S=Object.assign(Object.assign({},s),{style:Object.assign(Object.assign({},c),{margin:0}),open:b,visible:b,getPopupContainer:()=>d.current});return i&&(S=i(S)),a.createElement("div",{ref:d,style:{paddingBottom:u,position:"relative",minWidth:g}},a.createElement(e,Object.assign({},S)))})},99129:function(e,t,n){let a;n.d(t,{Z:function(){return eY}});var r=n(63787),i=n(2265),o=n(37274),s=n(57499),l=n(54165),c=n(99537),d=n(77136),u=n(20653),p=n(40388),g=n(16480),m=n.n(g),b=n(51761),f=n(47387),E=n(70595),h=n(24750),S=n(89211),y=n(1861),T=n(51350),A=e=>{let{type:t,children:n,prefixCls:a,buttonProps:r,close:o,autoFocus:s,emitEvent:l,isSilent:c,quitOnNullishReturnValue:d,actionFn:u}=e,p=i.useRef(!1),g=i.useRef(null),[m,b]=(0,S.Z)(!1),f=function(){null==o||o.apply(void 0,arguments)};i.useEffect(()=>{let e=null;return s&&(e=setTimeout(()=>{var e;null===(e=g.current)||void 0===e||e.focus()})),()=>{e&&clearTimeout(e)}},[]);let E=e=>{e&&e.then&&(b(!0),e.then(function(){b(!1,!0),f.apply(void 0,arguments),p.current=!1},e=>{if(b(!1,!0),p.current=!1,null==c||!c())return Promise.reject(e)}))};return i.createElement(y.ZP,Object.assign({},(0,T.nx)(t),{onClick:e=>{let t;if(!p.current){if(p.current=!0,!u){f();return}if(l){var n;if(t=u(e),d&&!((n=t)&&n.then)){p.current=!1,f(e);return}}else if(u.length)t=u(o),p.current=!1;else if(!(t=u())){f();return}E(t)}},loading:m,prefixCls:a},r,{ref:g}),n)};let R=i.createContext({}),{Provider:I}=R;var N=()=>{let{autoFocusButton:e,cancelButtonProps:t,cancelTextLocale:n,isSilent:a,mergedOkCancel:r,rootPrefixCls:o,close:s,onCancel:l,onConfirm:c}=(0,i.useContext)(R);return r?i.createElement(A,{isSilent:a,actionFn:l,close:function(){null==s||s.apply(void 0,arguments),null==c||c(!1)},autoFocus:"cancel"===e,buttonProps:t,prefixCls:"".concat(o,"-btn")},n):null},_=()=>{let{autoFocusButton:e,close:t,isSilent:n,okButtonProps:a,rootPrefixCls:r,okTextLocale:o,okType:s,onConfirm:l,onOk:c}=(0,i.useContext)(R);return i.createElement(A,{isSilent:n,type:s||"primary",actionFn:c,close:function(){null==t||t.apply(void 0,arguments),null==l||l(!0)},autoFocus:"ok"===e,buttonProps:a,prefixCls:"".concat(r,"-btn")},o)},v=n(81303),w=n(14749),k=n(80406),C=n(88804),O=i.createContext({}),x=n(5239),L=n(31506),D=n(91010),P=n(4295),M=n(72480);function F(e,t,n){var a=t;return!a&&n&&(a="".concat(e,"-").concat(n)),a}function U(e,t){var n=e["page".concat(t?"Y":"X","Offset")],a="scroll".concat(t?"Top":"Left");if("number"!=typeof n){var r=e.document;"number"!=typeof(n=r.documentElement[a])&&(n=r.body[a])}return n}var B=n(49367),G=n(74084),$=i.memo(function(e){return e.children},function(e,t){return!t.shouldUpdate}),H={width:0,height:0,overflow:"hidden",outline:"none"},z=i.forwardRef(function(e,t){var n,a,r,o=e.prefixCls,s=e.className,l=e.style,c=e.title,d=e.ariaId,u=e.footer,p=e.closable,g=e.closeIcon,b=e.onClose,f=e.children,E=e.bodyStyle,h=e.bodyProps,S=e.modalRender,y=e.onMouseDown,T=e.onMouseUp,A=e.holderRef,R=e.visible,I=e.forceRender,N=e.width,_=e.height,v=e.classNames,k=e.styles,C=i.useContext(O).panel,L=(0,G.x1)(A,C),D=(0,i.useRef)(),P=(0,i.useRef)();i.useImperativeHandle(t,function(){return{focus:function(){var e;null===(e=D.current)||void 0===e||e.focus()},changeActive:function(e){var t=document.activeElement;e&&t===P.current?D.current.focus():e||t!==D.current||P.current.focus()}}});var M={};void 0!==N&&(M.width=N),void 0!==_&&(M.height=_),u&&(n=i.createElement("div",{className:m()("".concat(o,"-footer"),null==v?void 0:v.footer),style:(0,x.Z)({},null==k?void 0:k.footer)},u)),c&&(a=i.createElement("div",{className:m()("".concat(o,"-header"),null==v?void 0:v.header),style:(0,x.Z)({},null==k?void 0:k.header)},i.createElement("div",{className:"".concat(o,"-title"),id:d},c))),p&&(r=i.createElement("button",{type:"button",onClick:b,"aria-label":"Close",className:"".concat(o,"-close")},g||i.createElement("span",{className:"".concat(o,"-close-x")})));var F=i.createElement("div",{className:m()("".concat(o,"-content"),null==v?void 0:v.content),style:null==k?void 0:k.content},r,a,i.createElement("div",(0,w.Z)({className:m()("".concat(o,"-body"),null==v?void 0:v.body),style:(0,x.Z)((0,x.Z)({},E),null==k?void 0:k.body)},h),f),n);return i.createElement("div",{key:"dialog-element",role:"dialog","aria-labelledby":c?d:null,"aria-modal":"true",ref:L,style:(0,x.Z)((0,x.Z)({},l),M),className:m()(o,s),onMouseDown:y,onMouseUp:T},i.createElement("div",{tabIndex:0,ref:D,style:H,"aria-hidden":"true"}),i.createElement($,{shouldUpdate:R||I},S?S(F):F),i.createElement("div",{tabIndex:0,ref:P,style:H,"aria-hidden":"true"}))}),j=i.forwardRef(function(e,t){var n=e.prefixCls,a=e.title,r=e.style,o=e.className,s=e.visible,l=e.forceRender,c=e.destroyOnClose,d=e.motionName,u=e.ariaId,p=e.onVisibleChanged,g=e.mousePosition,b=(0,i.useRef)(),f=i.useState(),E=(0,k.Z)(f,2),h=E[0],S=E[1],y={};function T(){var e,t,n,a,r,i=(n={left:(t=(e=b.current).getBoundingClientRect()).left,top:t.top},r=(a=e.ownerDocument).defaultView||a.parentWindow,n.left+=U(r),n.top+=U(r,!0),n);S(g?"".concat(g.x-i.left,"px ").concat(g.y-i.top,"px"):"")}return h&&(y.transformOrigin=h),i.createElement(B.ZP,{visible:s,onVisibleChanged:p,onAppearPrepare:T,onEnterPrepare:T,forceRender:l,motionName:d,removeOnLeave:c,ref:b},function(s,l){var c=s.className,d=s.style;return i.createElement(z,(0,w.Z)({},e,{ref:t,title:a,ariaId:u,prefixCls:n,holderRef:l,style:(0,x.Z)((0,x.Z)((0,x.Z)({},d),r),y),className:m()(o,c)}))})});function V(e){var t=e.prefixCls,n=e.style,a=e.visible,r=e.maskProps,o=e.motionName,s=e.className;return i.createElement(B.ZP,{key:"mask",visible:a,motionName:o,leavedClassName:"".concat(t,"-mask-hidden")},function(e,a){var o=e.className,l=e.style;return i.createElement("div",(0,w.Z)({ref:a,style:(0,x.Z)((0,x.Z)({},l),n),className:m()("".concat(t,"-mask"),o,s)},r))})}function W(e){var t=e.prefixCls,n=void 0===t?"rc-dialog":t,a=e.zIndex,r=e.visible,o=void 0!==r&&r,s=e.keyboard,l=void 0===s||s,c=e.focusTriggerAfterClose,d=void 0===c||c,u=e.wrapStyle,p=e.wrapClassName,g=e.wrapProps,b=e.onClose,f=e.afterOpenChange,E=e.afterClose,h=e.transitionName,S=e.animation,y=e.closable,T=e.mask,A=void 0===T||T,R=e.maskTransitionName,I=e.maskAnimation,N=e.maskClosable,_=e.maskStyle,v=e.maskProps,C=e.rootClassName,O=e.classNames,U=e.styles,B=(0,i.useRef)(),G=(0,i.useRef)(),$=(0,i.useRef)(),H=i.useState(o),z=(0,k.Z)(H,2),W=z[0],q=z[1],Y=(0,D.Z)();function K(e){null==b||b(e)}var Z=(0,i.useRef)(!1),X=(0,i.useRef)(),Q=null;return(void 0===N||N)&&(Q=function(e){Z.current?Z.current=!1:G.current===e.target&&K(e)}),(0,i.useEffect)(function(){o&&(q(!0),(0,L.Z)(G.current,document.activeElement)||(B.current=document.activeElement))},[o]),(0,i.useEffect)(function(){return function(){clearTimeout(X.current)}},[]),i.createElement("div",(0,w.Z)({className:m()("".concat(n,"-root"),C)},(0,M.Z)(e,{data:!0})),i.createElement(V,{prefixCls:n,visible:A&&o,motionName:F(n,R,I),style:(0,x.Z)((0,x.Z)({zIndex:a},_),null==U?void 0:U.mask),maskProps:v,className:null==O?void 0:O.mask}),i.createElement("div",(0,w.Z)({tabIndex:-1,onKeyDown:function(e){if(l&&e.keyCode===P.Z.ESC){e.stopPropagation(),K(e);return}o&&e.keyCode===P.Z.TAB&&$.current.changeActive(!e.shiftKey)},className:m()("".concat(n,"-wrap"),p,null==O?void 0:O.wrapper),ref:G,onClick:Q,style:(0,x.Z)((0,x.Z)((0,x.Z)({zIndex:a},u),null==U?void 0:U.wrapper),{},{display:W?null:"none"})},g),i.createElement(j,(0,w.Z)({},e,{onMouseDown:function(){clearTimeout(X.current),Z.current=!0},onMouseUp:function(){X.current=setTimeout(function(){Z.current=!1})},ref:$,closable:void 0===y||y,ariaId:Y,prefixCls:n,visible:o&&W,onClose:K,onVisibleChanged:function(e){if(e)!function(){if(!(0,L.Z)(G.current,document.activeElement)){var e;null===(e=$.current)||void 0===e||e.focus()}}();else{if(q(!1),A&&B.current&&d){try{B.current.focus({preventScroll:!0})}catch(e){}B.current=null}W&&(null==E||E())}null==f||f(e)},motionName:F(n,h,S)}))))}j.displayName="Content",n(53850);var q=function(e){var t=e.visible,n=e.getContainer,a=e.forceRender,r=e.destroyOnClose,o=void 0!==r&&r,s=e.afterClose,l=e.panelRef,c=i.useState(t),d=(0,k.Z)(c,2),u=d[0],p=d[1],g=i.useMemo(function(){return{panel:l}},[l]);return(i.useEffect(function(){t&&p(!0)},[t]),a||!o||u)?i.createElement(O.Provider,{value:g},i.createElement(C.Z,{open:t||a||u,autoDestroy:!1,getContainer:n,autoLock:t||u},i.createElement(W,(0,w.Z)({},e,{destroyOnClose:o,afterClose:function(){null==s||s(),p(!1)}})))):null};q.displayName="Dialog";var Y=function(e,t,n){let a=arguments.length>3&&void 0!==arguments[3]?arguments[3]:i.createElement(v.Z,null),r=arguments.length>4&&void 0!==arguments[4]&&arguments[4];if("boolean"==typeof e?!e:void 0===t?!r:!1===t||null===t)return[!1,null];let o="boolean"==typeof t||null==t?a:t;return[!0,n?n(o):o]},K=n(22127),Z=n(86718),X=n(47137),Q=n(92801),J=n(48563);function ee(){}let et=i.createContext({add:ee,remove:ee});var en=n(17094),ea=()=>{let{cancelButtonProps:e,cancelTextLocale:t,onCancel:n}=(0,i.useContext)(R);return i.createElement(y.ZP,Object.assign({onClick:n},e),t)},er=()=>{let{confirmLoading:e,okButtonProps:t,okType:n,okTextLocale:a,onOk:r}=(0,i.useContext)(R);return i.createElement(y.ZP,Object.assign({},(0,T.nx)(n),{loading:e,onClick:r},t),a)},ei=n(4678);function eo(e,t){return i.createElement("span",{className:"".concat(e,"-close-x")},t||i.createElement(v.Z,{className:"".concat(e,"-close-icon")}))}let es=e=>{let t;let{okText:n,okType:a="primary",cancelText:o,confirmLoading:s,onOk:l,onCancel:c,okButtonProps:d,cancelButtonProps:u,footer:p}=e,[g]=(0,E.Z)("Modal",(0,ei.A)()),m={confirmLoading:s,okButtonProps:d,cancelButtonProps:u,okTextLocale:n||(null==g?void 0:g.okText),cancelTextLocale:o||(null==g?void 0:g.cancelText),okType:a,onOk:l,onCancel:c},b=i.useMemo(()=>m,(0,r.Z)(Object.values(m)));return"function"==typeof p||void 0===p?(t=i.createElement(i.Fragment,null,i.createElement(ea,null),i.createElement(er,null)),"function"==typeof p&&(t=p(t,{OkBtn:er,CancelBtn:ea})),t=i.createElement(I,{value:b},t)):t=p,i.createElement(en.n,{disabled:!1},t)};var el=n(11303),ec=n(13703),ed=n(58854),eu=n(80316),ep=n(76585),eg=n(8985);function em(e){return{position:e,inset:0}}let eb=e=>{let{componentCls:t,antCls:n}=e;return[{["".concat(t,"-root")]:{["".concat(t).concat(n,"-zoom-enter, ").concat(t).concat(n,"-zoom-appear")]:{transform:"none",opacity:0,animationDuration:e.motionDurationSlow,userSelect:"none"},["".concat(t).concat(n,"-zoom-leave ").concat(t,"-content")]:{pointerEvents:"none"},["".concat(t,"-mask")]:Object.assign(Object.assign({},em("fixed")),{zIndex:e.zIndexPopupBase,height:"100%",backgroundColor:e.colorBgMask,pointerEvents:"none",["".concat(t,"-hidden")]:{display:"none"}}),["".concat(t,"-wrap")]:Object.assign(Object.assign({},em("fixed")),{zIndex:e.zIndexPopupBase,overflow:"auto",outline:0,WebkitOverflowScrolling:"touch",["&:has(".concat(t).concat(n,"-zoom-enter), &:has(").concat(t).concat(n,"-zoom-appear)")]:{pointerEvents:"none"}})}},{["".concat(t,"-root")]:(0,ec.J$)(e)}]},ef=e=>{let{componentCls:t}=e;return[{["".concat(t,"-root")]:{["".concat(t,"-wrap-rtl")]:{direction:"rtl"},["".concat(t,"-centered")]:{textAlign:"center","&::before":{display:"inline-block",width:0,height:"100%",verticalAlign:"middle",content:'""'},[t]:{top:0,display:"inline-block",paddingBottom:0,textAlign:"start",verticalAlign:"middle"}},["@media (max-width: ".concat(e.screenSMMax,"px)")]:{[t]:{maxWidth:"calc(100vw - 16px)",margin:"".concat((0,eg.bf)(e.marginXS)," auto")},["".concat(t,"-centered")]:{[t]:{flex:1}}}}},{[t]:Object.assign(Object.assign({},(0,el.Wf)(e)),{pointerEvents:"none",position:"relative",top:100,width:"auto",maxWidth:"calc(100vw - ".concat((0,eg.bf)(e.calc(e.margin).mul(2).equal()),")"),margin:"0 auto",paddingBottom:e.paddingLG,["".concat(t,"-title")]:{margin:0,color:e.titleColor,fontWeight:e.fontWeightStrong,fontSize:e.titleFontSize,lineHeight:e.titleLineHeight,wordWrap:"break-word"},["".concat(t,"-content")]:{position:"relative",backgroundColor:e.contentBg,backgroundClip:"padding-box",border:0,borderRadius:e.borderRadiusLG,boxShadow:e.boxShadow,pointerEvents:"auto",padding:e.contentPadding},["".concat(t,"-close")]:Object.assign({position:"absolute",top:e.calc(e.modalHeaderHeight).sub(e.modalCloseBtnSize).div(2).equal(),insetInlineEnd:e.calc(e.modalHeaderHeight).sub(e.modalCloseBtnSize).div(2).equal(),zIndex:e.calc(e.zIndexPopupBase).add(10).equal(),padding:0,color:e.modalCloseIconColor,fontWeight:e.fontWeightStrong,lineHeight:1,textDecoration:"none",background:"transparent",borderRadius:e.borderRadiusSM,width:e.modalCloseBtnSize,height:e.modalCloseBtnSize,border:0,outline:0,cursor:"pointer",transition:"color ".concat(e.motionDurationMid,", background-color ").concat(e.motionDurationMid),"&-x":{display:"flex",fontSize:e.fontSizeLG,fontStyle:"normal",lineHeight:"".concat((0,eg.bf)(e.modalCloseBtnSize)),justifyContent:"center",textTransform:"none",textRendering:"auto"},"&:hover":{color:e.modalIconHoverColor,backgroundColor:e.closeBtnHoverBg,textDecoration:"none"},"&:active":{backgroundColor:e.closeBtnActiveBg}},(0,el.Qy)(e)),["".concat(t,"-header")]:{color:e.colorText,background:e.headerBg,borderRadius:"".concat((0,eg.bf)(e.borderRadiusLG)," ").concat((0,eg.bf)(e.borderRadiusLG)," 0 0"),marginBottom:e.headerMarginBottom,padding:e.headerPadding,borderBottom:e.headerBorderBottom},["".concat(t,"-body")]:{fontSize:e.fontSize,lineHeight:e.lineHeight,wordWrap:"break-word",padding:e.bodyPadding},["".concat(t,"-footer")]:{textAlign:"end",background:e.footerBg,marginTop:e.footerMarginTop,padding:e.footerPadding,borderTop:e.footerBorderTop,borderRadius:e.footerBorderRadius,["> ".concat(e.antCls,"-btn + ").concat(e.antCls,"-btn")]:{marginInlineStart:e.marginXS}},["".concat(t,"-open")]:{overflow:"hidden"}})},{["".concat(t,"-pure-panel")]:{top:"auto",padding:0,display:"flex",flexDirection:"column",["".concat(t,"-content,\n ").concat(t,"-body,\n ").concat(t,"-confirm-body-wrapper")]:{display:"flex",flexDirection:"column",flex:"auto"},["".concat(t,"-confirm-body")]:{marginBottom:"auto"}}}]},eE=e=>{let{componentCls:t}=e;return{["".concat(t,"-root")]:{["".concat(t,"-wrap-rtl")]:{direction:"rtl",["".concat(t,"-confirm-body")]:{direction:"rtl"}}}}},eh=e=>{let t=e.padding,n=e.fontSizeHeading5,a=e.lineHeightHeading5;return(0,eu.TS)(e,{modalHeaderHeight:e.calc(e.calc(a).mul(n).equal()).add(e.calc(t).mul(2).equal()).equal(),modalFooterBorderColorSplit:e.colorSplit,modalFooterBorderStyle:e.lineType,modalFooterBorderWidth:e.lineWidth,modalIconHoverColor:e.colorIconHover,modalCloseIconColor:e.colorIcon,modalCloseBtnSize:e.fontHeight,modalConfirmIconSize:e.fontHeight,modalTitleHeight:e.calc(e.titleFontSize).mul(e.titleLineHeight).equal()})},eS=e=>({footerBg:"transparent",headerBg:e.colorBgElevated,titleLineHeight:e.lineHeightHeading5,titleFontSize:e.fontSizeHeading5,contentBg:e.colorBgElevated,titleColor:e.colorTextHeading,closeBtnHoverBg:e.wireframe?"transparent":e.colorFillContent,closeBtnActiveBg:e.wireframe?"transparent":e.colorFillContentHover,contentPadding:e.wireframe?0:"".concat((0,eg.bf)(e.paddingMD)," ").concat((0,eg.bf)(e.paddingContentHorizontalLG)),headerPadding:e.wireframe?"".concat((0,eg.bf)(e.padding)," ").concat((0,eg.bf)(e.paddingLG)):0,headerBorderBottom:e.wireframe?"".concat((0,eg.bf)(e.lineWidth)," ").concat(e.lineType," ").concat(e.colorSplit):"none",headerMarginBottom:e.wireframe?0:e.marginXS,bodyPadding:e.wireframe?e.paddingLG:0,footerPadding:e.wireframe?"".concat((0,eg.bf)(e.paddingXS)," ").concat((0,eg.bf)(e.padding)):0,footerBorderTop:e.wireframe?"".concat((0,eg.bf)(e.lineWidth)," ").concat(e.lineType," ").concat(e.colorSplit):"none",footerBorderRadius:e.wireframe?"0 0 ".concat((0,eg.bf)(e.borderRadiusLG)," ").concat((0,eg.bf)(e.borderRadiusLG)):0,footerMarginTop:e.wireframe?0:e.marginSM,confirmBodyPadding:e.wireframe?"".concat((0,eg.bf)(2*e.padding)," ").concat((0,eg.bf)(2*e.padding)," ").concat((0,eg.bf)(e.paddingLG)):0,confirmIconMarginInlineEnd:e.wireframe?e.margin:e.marginSM,confirmBtnsMarginTop:e.wireframe?e.marginLG:e.marginSM});var ey=(0,ep.I$)("Modal",e=>{let t=eh(e);return[ef(t),eE(t),eb(t),(0,ed._y)(t,"zoom")]},eS,{unitless:{titleLineHeight:!0}}),eT=n(92935),eA=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n};(0,K.Z)()&&window.document.documentElement&&document.documentElement.addEventListener("click",e=>{a={x:e.pageX,y:e.pageY},setTimeout(()=>{a=null},100)},!0);var eR=e=>{var t;let{getPopupContainer:n,getPrefixCls:r,direction:o,modal:l}=i.useContext(s.E_),c=t=>{let{onCancel:n}=e;null==n||n(t)},{prefixCls:d,className:u,rootClassName:p,open:g,wrapClassName:E,centered:h,getContainer:S,closeIcon:y,closable:T,focusTriggerAfterClose:A=!0,style:R,visible:I,width:N=520,footer:_,classNames:w,styles:k}=e,C=eA(e,["prefixCls","className","rootClassName","open","wrapClassName","centered","getContainer","closeIcon","closable","focusTriggerAfterClose","style","visible","width","footer","classNames","styles"]),O=r("modal",d),x=r(),L=(0,eT.Z)(O),[D,P,M]=ey(O,L),F=m()(E,{["".concat(O,"-centered")]:!!h,["".concat(O,"-wrap-rtl")]:"rtl"===o}),U=null!==_&&i.createElement(es,Object.assign({},e,{onOk:t=>{let{onOk:n}=e;null==n||n(t)},onCancel:c})),[B,G]=Y(T,y,e=>eo(O,e),i.createElement(v.Z,{className:"".concat(O,"-close-icon")}),!0),$=function(e){let t=i.useContext(et),n=i.useRef();return(0,J.zX)(a=>{if(a){let r=e?a.querySelector(e):a;t.add(r),n.current=r}else t.remove(n.current)})}(".".concat(O,"-content")),[H,z]=(0,b.Cn)("Modal",C.zIndex);return D(i.createElement(Q.BR,null,i.createElement(X.Ux,{status:!0,override:!0},i.createElement(Z.Z.Provider,{value:z},i.createElement(q,Object.assign({width:N},C,{zIndex:H,getContainer:void 0===S?n:S,prefixCls:O,rootClassName:m()(P,p,M,L),footer:U,visible:null!=g?g:I,mousePosition:null!==(t=C.mousePosition)&&void 0!==t?t:a,onClose:c,closable:B,closeIcon:G,focusTriggerAfterClose:A,transitionName:(0,f.m)(x,"zoom",e.transitionName),maskTransitionName:(0,f.m)(x,"fade",e.maskTransitionName),className:m()(P,u,null==l?void 0:l.className),style:Object.assign(Object.assign({},null==l?void 0:l.style),R),classNames:Object.assign(Object.assign({wrapper:F},null==l?void 0:l.classNames),w),styles:Object.assign(Object.assign({},null==l?void 0:l.styles),k),panelRef:$}))))))};let eI=e=>{let{componentCls:t,titleFontSize:n,titleLineHeight:a,modalConfirmIconSize:r,fontSize:i,lineHeight:o,modalTitleHeight:s,fontHeight:l,confirmBodyPadding:c}=e,d="".concat(t,"-confirm");return{[d]:{"&-rtl":{direction:"rtl"},["".concat(e.antCls,"-modal-header")]:{display:"none"},["".concat(d,"-body-wrapper")]:Object.assign({},(0,el.dF)()),["&".concat(t," ").concat(t,"-body")]:{padding:c},["".concat(d,"-body")]:{display:"flex",flexWrap:"nowrap",alignItems:"start",["> ".concat(e.iconCls)]:{flex:"none",fontSize:r,marginInlineEnd:e.confirmIconMarginInlineEnd,marginTop:e.calc(e.calc(l).sub(r).equal()).div(2).equal()},["&-has-title > ".concat(e.iconCls)]:{marginTop:e.calc(e.calc(s).sub(r).equal()).div(2).equal()}},["".concat(d,"-paragraph")]:{display:"flex",flexDirection:"column",flex:"auto",rowGap:e.marginXS,maxWidth:"calc(100% - ".concat((0,eg.bf)(e.calc(e.modalConfirmIconSize).add(e.marginSM).equal()),")")},["".concat(d,"-title")]:{color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:n,lineHeight:a},["".concat(d,"-content")]:{color:e.colorText,fontSize:i,lineHeight:o},["".concat(d,"-btns")]:{textAlign:"end",marginTop:e.confirmBtnsMarginTop,["".concat(e.antCls,"-btn + ").concat(e.antCls,"-btn")]:{marginBottom:0,marginInlineStart:e.marginXS}}},["".concat(d,"-error ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorError},["".concat(d,"-warning ").concat(d,"-body > ").concat(e.iconCls,",\n ").concat(d,"-confirm ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorWarning},["".concat(d,"-info ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorInfo},["".concat(d,"-success ").concat(d,"-body > ").concat(e.iconCls)]:{color:e.colorSuccess}}};var eN=(0,ep.bk)(["Modal","confirm"],e=>[eI(eh(e))],eS,{order:-1e3}),e_=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n};function ev(e){let{prefixCls:t,icon:n,okText:a,cancelText:o,confirmPrefixCls:s,type:l,okCancel:g,footer:b,locale:f}=e,h=e_(e,["prefixCls","icon","okText","cancelText","confirmPrefixCls","type","okCancel","footer","locale"]),S=n;if(!n&&null!==n)switch(l){case"info":S=i.createElement(p.Z,null);break;case"success":S=i.createElement(c.Z,null);break;case"error":S=i.createElement(d.Z,null);break;default:S=i.createElement(u.Z,null)}let y=null!=g?g:"confirm"===l,T=null!==e.autoFocusButton&&(e.autoFocusButton||"ok"),[A]=(0,E.Z)("Modal"),R=f||A,v=a||(y?null==R?void 0:R.okText:null==R?void 0:R.justOkText),w=Object.assign({autoFocusButton:T,cancelTextLocale:o||(null==R?void 0:R.cancelText),okTextLocale:v,mergedOkCancel:y},h),k=i.useMemo(()=>w,(0,r.Z)(Object.values(w))),C=i.createElement(i.Fragment,null,i.createElement(N,null),i.createElement(_,null)),O=void 0!==e.title&&null!==e.title,x="".concat(s,"-body");return i.createElement("div",{className:"".concat(s,"-body-wrapper")},i.createElement("div",{className:m()(x,{["".concat(x,"-has-title")]:O})},S,i.createElement("div",{className:"".concat(s,"-paragraph")},O&&i.createElement("span",{className:"".concat(s,"-title")},e.title),i.createElement("div",{className:"".concat(s,"-content")},e.content))),void 0===b||"function"==typeof b?i.createElement(I,{value:k},i.createElement("div",{className:"".concat(s,"-btns")},"function"==typeof b?b(C,{OkBtn:_,CancelBtn:N}):C)):b,i.createElement(eN,{prefixCls:t}))}let ew=e=>{let{close:t,zIndex:n,afterClose:a,open:r,keyboard:o,centered:s,getContainer:l,maskStyle:c,direction:d,prefixCls:u,wrapClassName:p,rootPrefixCls:g,bodyStyle:E,closable:S=!1,closeIcon:y,modalRender:T,focusTriggerAfterClose:A,onConfirm:R,styles:I}=e,N="".concat(u,"-confirm"),_=e.width||416,v=e.style||{},w=void 0===e.mask||e.mask,k=void 0!==e.maskClosable&&e.maskClosable,C=m()(N,"".concat(N,"-").concat(e.type),{["".concat(N,"-rtl")]:"rtl"===d},e.className),[,O]=(0,h.ZP)(),x=i.useMemo(()=>void 0!==n?n:O.zIndexPopupBase+b.u6,[n,O]);return i.createElement(eR,{prefixCls:u,className:C,wrapClassName:m()({["".concat(N,"-centered")]:!!e.centered},p),onCancel:()=>{null==t||t({triggerCancel:!0}),null==R||R(!1)},open:r,title:"",footer:null,transitionName:(0,f.m)(g||"","zoom",e.transitionName),maskTransitionName:(0,f.m)(g||"","fade",e.maskTransitionName),mask:w,maskClosable:k,style:v,styles:Object.assign({body:E,mask:c},I),width:_,zIndex:x,afterClose:a,keyboard:o,centered:s,getContainer:l,closable:S,closeIcon:y,modalRender:T,focusTriggerAfterClose:A},i.createElement(ev,Object.assign({},e,{confirmPrefixCls:N})))};var ek=e=>{let{rootPrefixCls:t,iconPrefixCls:n,direction:a,theme:r}=e;return i.createElement(l.ZP,{prefixCls:t,iconPrefixCls:n,direction:a,theme:r},i.createElement(ew,Object.assign({},e)))},eC=[];let eO="",ex=e=>{var t,n;let{prefixCls:a,getContainer:r,direction:o}=e,l=(0,ei.A)(),c=(0,i.useContext)(s.E_),d=eO||c.getPrefixCls(),u=a||"".concat(d,"-modal"),p=r;return!1===p&&(p=void 0),i.createElement(ek,Object.assign({},e,{rootPrefixCls:d,prefixCls:u,iconPrefixCls:c.iconPrefixCls,theme:c.theme,direction:null!=o?o:c.direction,locale:null!==(n=null===(t=c.locale)||void 0===t?void 0:t.Modal)&&void 0!==n?n:l,getContainer:p}))};function eL(e){let t;let n=(0,l.w6)(),a=document.createDocumentFragment(),s=Object.assign(Object.assign({},e),{close:u,open:!0});function c(){for(var t=arguments.length,n=Array(t),i=0;ie&&e.triggerCancel);e.onCancel&&s&&e.onCancel.apply(e,[()=>{}].concat((0,r.Z)(n.slice(1))));for(let e=0;e{let t=n.getPrefixCls(void 0,eO),r=n.getIconPrefixCls(),s=n.getTheme(),c=i.createElement(ex,Object.assign({},e));(0,o.s)(i.createElement(l.ZP,{prefixCls:t,iconPrefixCls:r,theme:s},n.holderRender?n.holderRender(c):c),a)})}function u(){for(var t=arguments.length,n=Array(t),a=0;a{"function"==typeof e.afterClose&&e.afterClose(),c.apply(this,n)}})).visible&&delete s.visible,d(s)}return d(s),eC.push(u),{destroy:u,update:function(e){d(s="function"==typeof e?e(s):Object.assign(Object.assign({},s),e))}}}function eD(e){return Object.assign(Object.assign({},e),{type:"warning"})}function eP(e){return Object.assign(Object.assign({},e),{type:"info"})}function eM(e){return Object.assign(Object.assign({},e),{type:"success"})}function eF(e){return Object.assign(Object.assign({},e),{type:"error"})}function eU(e){return Object.assign(Object.assign({},e),{type:"confirm"})}var eB=n(21467),eG=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n},e$=(0,eB.i)(e=>{let{prefixCls:t,className:n,closeIcon:a,closable:r,type:o,title:l,children:c,footer:d}=e,u=eG(e,["prefixCls","className","closeIcon","closable","type","title","children","footer"]),{getPrefixCls:p}=i.useContext(s.E_),g=p(),b=t||p("modal"),f=(0,eT.Z)(g),[E,h,S]=ey(b,f),y="".concat(b,"-confirm"),T={};return T=o?{closable:null!=r&&r,title:"",footer:"",children:i.createElement(ev,Object.assign({},e,{prefixCls:b,confirmPrefixCls:y,rootPrefixCls:g,content:c}))}:{closable:null==r||r,title:l,footer:null!==d&&i.createElement(es,Object.assign({},e)),children:c},E(i.createElement(z,Object.assign({prefixCls:b,className:m()(h,"".concat(b,"-pure-panel"),o&&y,o&&"".concat(y,"-").concat(o),n,S,f)},u,{closeIcon:eo(b,a),closable:r},T)))}),eH=n(79474),ez=function(e,t){var n={};for(var a in e)Object.prototype.hasOwnProperty.call(e,a)&&0>t.indexOf(a)&&(n[a]=e[a]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var r=0,a=Object.getOwnPropertySymbols(e);rt.indexOf(a[r])&&Object.prototype.propertyIsEnumerable.call(e,a[r])&&(n[a[r]]=e[a[r]]);return n},ej=i.forwardRef((e,t)=>{var n,{afterClose:a,config:o}=e,l=ez(e,["afterClose","config"]);let[c,d]=i.useState(!0),[u,p]=i.useState(o),{direction:g,getPrefixCls:m}=i.useContext(s.E_),b=m("modal"),f=m(),h=function(){d(!1);for(var e=arguments.length,t=Array(e),n=0;ne&&e.triggerCancel);u.onCancel&&a&&u.onCancel.apply(u,[()=>{}].concat((0,r.Z)(t.slice(1))))};i.useImperativeHandle(t,()=>({destroy:h,update:e=>{p(t=>Object.assign(Object.assign({},t),e))}}));let S=null!==(n=u.okCancel)&&void 0!==n?n:"confirm"===u.type,[y]=(0,E.Z)("Modal",eH.Z.Modal);return i.createElement(ek,Object.assign({prefixCls:b,rootPrefixCls:f},u,{close:h,open:c,afterClose:()=>{var e;a(),null===(e=u.afterClose)||void 0===e||e.call(u)},okText:u.okText||(S?null==y?void 0:y.okText:null==y?void 0:y.justOkText),direction:u.direction||g,cancelText:u.cancelText||(null==y?void 0:y.cancelText)},l))});let eV=0,eW=i.memo(i.forwardRef((e,t)=>{let[n,a]=function(){let[e,t]=i.useState([]);return[e,i.useCallback(e=>(t(t=>[].concat((0,r.Z)(t),[e])),()=>{t(t=>t.filter(t=>t!==e))}),[])]}();return i.useImperativeHandle(t,()=>({patchElement:a}),[]),i.createElement(i.Fragment,null,n)}));function eq(e){return eL(eD(e))}eR.useModal=function(){let e=i.useRef(null),[t,n]=i.useState([]);i.useEffect(()=>{t.length&&((0,r.Z)(t).forEach(e=>{e()}),n([]))},[t]);let a=i.useCallback(t=>function(a){var o;let s,l;eV+=1;let c=i.createRef(),d=new Promise(e=>{s=e}),u=!1,p=i.createElement(ej,{key:"modal-".concat(eV),config:t(a),ref:c,afterClose:()=>{null==l||l()},isSilent:()=>u,onConfirm:e=>{s(e)}});return(l=null===(o=e.current)||void 0===o?void 0:o.patchElement(p))&&eC.push(l),{destroy:()=>{function e(){var e;null===(e=c.current)||void 0===e||e.destroy()}c.current?e():n(t=>[].concat((0,r.Z)(t),[e]))},update:e=>{function t(){var t;null===(t=c.current)||void 0===t||t.update(e)}c.current?t():n(e=>[].concat((0,r.Z)(e),[t]))},then:e=>(u=!0,d.then(e))}},[]);return[i.useMemo(()=>({info:a(eP),success:a(eM),error:a(eF),warning:a(eD),confirm:a(eU)}),[]),i.createElement(eW,{key:"modal-holder",ref:e})]},eR.info=function(e){return eL(eP(e))},eR.success=function(e){return eL(eM(e))},eR.error=function(e){return eL(eF(e))},eR.warning=eq,eR.warn=eq,eR.confirm=function(e){return eL(eU(e))},eR.destroyAll=function(){for(;eC.length;){let e=eC.pop();e&&e()}},eR.config=function(e){let{rootPrefixCls:t}=e;eO=t},eR._InternalPanelDoNotUseOrYouWillBeFired=e$;var eY=eR},13703:function(e,t,n){n.d(t,{J$:function(){return s}});var a=n(8985),r=n(59353);let i=new a.E4("antFadeIn",{"0%":{opacity:0},"100%":{opacity:1}}),o=new a.E4("antFadeOut",{"0%":{opacity:1},"100%":{opacity:0}}),s=function(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],{antCls:n}=e,a="".concat(n,"-fade"),s=t?"&":"";return[(0,r.R)(a,i,o,e.motionDurationMid,t),{["\n ".concat(s).concat(a,"-enter,\n ").concat(s).concat(a,"-appear\n ")]:{opacity:0,animationTimingFunction:"linear"},["".concat(s).concat(a,"-leave")]:{animationTimingFunction:"linear"}}]}},44056:function(e){e.exports=function(e,n){for(var a,r,i,o=e||"",s=n||"div",l={},c=0;c4&&m.slice(0,4)===o&&s.test(t)&&("-"===t.charAt(4)?b=o+(n=t.slice(5).replace(l,u)).charAt(0).toUpperCase()+n.slice(1):(g=(p=t).slice(4),t=l.test(g)?p:("-"!==(g=g.replace(c,d)).charAt(0)&&(g="-"+g),o+g)),f=r),new f(b,t))};var s=/^data[-\w.:]+$/i,l=/-[a-z]/g,c=/[A-Z]/g;function d(e){return"-"+e.toLowerCase()}function u(e){return e.charAt(1).toUpperCase()}},31872:function(e,t,n){var a=n(96130),r=n(64730),i=n(61861),o=n(46982),s=n(83671),l=n(53618);e.exports=a([i,r,o,s,l])},83671:function(e,t,n){var a=n(7667),r=n(13585),i=a.booleanish,o=a.number,s=a.spaceSeparated;e.exports=r({transform:function(e,t){return"role"===t?t:"aria-"+t.slice(4).toLowerCase()},properties:{ariaActiveDescendant:null,ariaAtomic:i,ariaAutoComplete:null,ariaBusy:i,ariaChecked:i,ariaColCount:o,ariaColIndex:o,ariaColSpan:o,ariaControls:s,ariaCurrent:null,ariaDescribedBy:s,ariaDetails:null,ariaDisabled:i,ariaDropEffect:s,ariaErrorMessage:null,ariaExpanded:i,ariaFlowTo:s,ariaGrabbed:i,ariaHasPopup:null,ariaHidden:i,ariaInvalid:null,ariaKeyShortcuts:null,ariaLabel:null,ariaLabelledBy:s,ariaLevel:o,ariaLive:null,ariaModal:i,ariaMultiLine:i,ariaMultiSelectable:i,ariaOrientation:null,ariaOwns:s,ariaPlaceholder:null,ariaPosInSet:o,ariaPressed:i,ariaReadOnly:i,ariaRelevant:null,ariaRequired:i,ariaRoleDescription:s,ariaRowCount:o,ariaRowIndex:o,ariaRowSpan:o,ariaSelected:i,ariaSetSize:o,ariaSort:null,ariaValueMax:o,ariaValueMin:o,ariaValueNow:o,ariaValueText:null,role:null}})},53618:function(e,t,n){var a=n(7667),r=n(13585),i=n(46640),o=a.boolean,s=a.overloadedBoolean,l=a.booleanish,c=a.number,d=a.spaceSeparated,u=a.commaSeparated;e.exports=r({space:"html",attributes:{acceptcharset:"accept-charset",classname:"class",htmlfor:"for",httpequiv:"http-equiv"},transform:i,mustUseProperty:["checked","multiple","muted","selected"],properties:{abbr:null,accept:u,acceptCharset:d,accessKey:d,action:null,allow:null,allowFullScreen:o,allowPaymentRequest:o,allowUserMedia:o,alt:null,as:null,async:o,autoCapitalize:null,autoComplete:d,autoFocus:o,autoPlay:o,capture:o,charSet:null,checked:o,cite:null,className:d,cols:c,colSpan:null,content:null,contentEditable:l,controls:o,controlsList:d,coords:c|u,crossOrigin:null,data:null,dateTime:null,decoding:null,default:o,defer:o,dir:null,dirName:null,disabled:o,download:s,draggable:l,encType:null,enterKeyHint:null,form:null,formAction:null,formEncType:null,formMethod:null,formNoValidate:o,formTarget:null,headers:d,height:c,hidden:o,high:c,href:null,hrefLang:null,htmlFor:d,httpEquiv:d,id:null,imageSizes:null,imageSrcSet:u,inputMode:null,integrity:null,is:null,isMap:o,itemId:null,itemProp:d,itemRef:d,itemScope:o,itemType:d,kind:null,label:null,lang:null,language:null,list:null,loading:null,loop:o,low:c,manifest:null,max:null,maxLength:c,media:null,method:null,min:null,minLength:c,multiple:o,muted:o,name:null,nonce:null,noModule:o,noValidate:o,onAbort:null,onAfterPrint:null,onAuxClick:null,onBeforePrint:null,onBeforeUnload:null,onBlur:null,onCancel:null,onCanPlay:null,onCanPlayThrough:null,onChange:null,onClick:null,onClose:null,onContextMenu:null,onCopy:null,onCueChange:null,onCut:null,onDblClick:null,onDrag:null,onDragEnd:null,onDragEnter:null,onDragExit:null,onDragLeave:null,onDragOver:null,onDragStart:null,onDrop:null,onDurationChange:null,onEmptied:null,onEnded:null,onError:null,onFocus:null,onFormData:null,onHashChange:null,onInput:null,onInvalid:null,onKeyDown:null,onKeyPress:null,onKeyUp:null,onLanguageChange:null,onLoad:null,onLoadedData:null,onLoadedMetadata:null,onLoadEnd:null,onLoadStart:null,onMessage:null,onMessageError:null,onMouseDown:null,onMouseEnter:null,onMouseLeave:null,onMouseMove:null,onMouseOut:null,onMouseOver:null,onMouseUp:null,onOffline:null,onOnline:null,onPageHide:null,onPageShow:null,onPaste:null,onPause:null,onPlay:null,onPlaying:null,onPopState:null,onProgress:null,onRateChange:null,onRejectionHandled:null,onReset:null,onResize:null,onScroll:null,onSecurityPolicyViolation:null,onSeeked:null,onSeeking:null,onSelect:null,onSlotChange:null,onStalled:null,onStorage:null,onSubmit:null,onSuspend:null,onTimeUpdate:null,onToggle:null,onUnhandledRejection:null,onUnload:null,onVolumeChange:null,onWaiting:null,onWheel:null,open:o,optimum:c,pattern:null,ping:d,placeholder:null,playsInline:o,poster:null,preload:null,readOnly:o,referrerPolicy:null,rel:d,required:o,reversed:o,rows:c,rowSpan:c,sandbox:d,scope:null,scoped:o,seamless:o,selected:o,shape:null,size:c,sizes:null,slot:null,span:c,spellCheck:l,src:null,srcDoc:null,srcLang:null,srcSet:u,start:c,step:null,style:null,tabIndex:c,target:null,title:null,translate:null,type:null,typeMustMatch:o,useMap:null,value:l,width:c,wrap:null,align:null,aLink:null,archive:d,axis:null,background:null,bgColor:null,border:c,borderColor:null,bottomMargin:c,cellPadding:null,cellSpacing:null,char:null,charOff:null,classId:null,clear:null,code:null,codeBase:null,codeType:null,color:null,compact:o,declare:o,event:null,face:null,frame:null,frameBorder:null,hSpace:c,leftMargin:c,link:null,longDesc:null,lowSrc:null,marginHeight:c,marginWidth:c,noResize:o,noHref:o,noShade:o,noWrap:o,object:null,profile:null,prompt:null,rev:null,rightMargin:c,rules:null,scheme:null,scrolling:l,standby:null,summary:null,text:null,topMargin:c,valueType:null,version:null,vAlign:null,vLink:null,vSpace:c,allowTransparency:null,autoCorrect:null,autoSave:null,disablePictureInPicture:o,disableRemotePlayback:o,prefix:null,property:null,results:c,security:null,unselectable:null}})},46640:function(e,t,n){var a=n(25852);e.exports=function(e,t){return a(e,t.toLowerCase())}},25852:function(e){e.exports=function(e,t){return t in e?e[t]:t}},13585:function(e,t,n){var a=n(39900),r=n(94949),i=n(7478);e.exports=function(e){var t,n,o=e.space,s=e.mustUseProperty||[],l=e.attributes||{},c=e.properties,d=e.transform,u={},p={};for(t in c)n=new i(t,d(l,t),c[t],o),-1!==s.indexOf(t)&&(n.mustUseProperty=!0),u[t]=n,p[a(t)]=t,p[a(n.attribute)]=t;return new r(u,p,o)}},7478:function(e,t,n){var a=n(74108),r=n(7667);e.exports=s,s.prototype=new a,s.prototype.defined=!0;var i=["boolean","booleanish","overloadedBoolean","number","commaSeparated","spaceSeparated","commaOrSpaceSeparated"],o=i.length;function s(e,t,n,s){var l,c,d,u=-1;for(s&&(this.space=s),a.call(this,e,t);++u