From 4e63bca21e6cd617f5ae11716bda963d055c14ee Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 12:52:20 -0800 Subject: [PATCH 01/26] (feat) clickhouse run view setup --- litellm/integrations/clickhouse.py | 110 +++++++++++++++++++++++++++++ litellm/proxy/proxy_config.yaml | 2 +- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 280d64c4a..1533dd6c6 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -27,6 +27,111 @@ import litellm, uuid from litellm._logging import print_verbose, verbose_logger +def _create_clickhouse_material_views(client=None, table_names=[]): + # Create Materialized Views if they don't exist + # Materialized Views send new inserted rows to the aggregate tables + + verbose_logger.debug("Clickhouse: Creating Materialized Views") + if "daily_aggregated_spend_per_model_mv" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_model_mv") + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_per_model_mv + TO daily_aggregated_spend_per_model + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend, + model as model + FROM spend_logs + GROUP BY + day, model + """ + ) + if "daily_aggregated_spend_per_api_key_mv" not in table_names: + verbose_logger.debug( + "Clickhouse: Creating daily_aggregated_spend_per_api_key_mv" + ) + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_per_api_key_mv + TO daily_aggregated_spend_per_api_key + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend, + api_key as api_key + FROM spend_logs + GROUP BY + day, api_key + """ + ) + if "daily_aggregated_spend_mv" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_mv") + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_mv + TO daily_aggregated_spend + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend + FROM spend_logs + GROUP BY + day + """ + ) + + +def _create_clickhouse_aggregate_tables(client=None, table_names=[]): + # Basic Logging works without this - this is only used for low latency reporting apis + verbose_logger.debug("Clickhouse: Creating Aggregate Tables") + + # Create Aggregeate Tables if they don't exist + if "daily_aggregated_spend_per_model" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_model") + client.command( + """ + CREATE TABLE daily_aggregated_spend_per_model + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + `model` String + ) + ENGINE = SummingMergeTree() + ORDER BY (day, model); + """ + ) + if "daily_aggregated_spend_per_api_key" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_api_key") + client.command( + """ + CREATE TABLE daily_aggregated_spend_per_api_key + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + `api_key` String + ) + ENGINE = SummingMergeTree() + ORDER BY (day, api_key); + """ + ) + if "daily_aggregated_spend" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend") + client.command( + """ + CREATE TABLE daily_aggregated_spend + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + ) + ENGINE = SummingMergeTree() + ORDER BY (day); + """ + ) + return + + def _start_clickhouse(): import clickhouse_connect @@ -86,6 +191,11 @@ def _start_clickhouse(): response = client.query("DESCRIBE default.spend_logs") verbose_logger.debug(f"spend logs schema ={response.result_rows}") + # RUN Enterprise Clickhouse Setup + # TLDR: For Enterprise - we create views / aggregate tables for low latency reporting APIs + _create_clickhouse_aggregate_tables(client=client, table_names=table_names) + _create_clickhouse_material_views(client=client, table_names=table_names) + class ClickhouseLogger: # Class variables or attributes diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 74a780c71..5755293e0 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -43,7 +43,7 @@ model_list: api_key: os.environ/OPENAI_API_KEY litellm_settings: fallbacks: [{"openai-gpt-3.5": ["azure-gpt-3.5"]}] - success_callback: ['langfuse'] + success_callback: ['clickhouse', 'langfuse'] # setting callback class # callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] From 16ff445e7a321405219c687a2d3e6c61853b96ac Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 12:55:50 -0800 Subject: [PATCH 02/26] (feat) clickhouse update views / aggregate tables --- litellm/integrations/clickhouse.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 1533dd6c6..82ecd3256 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -66,6 +66,22 @@ def _create_clickhouse_material_views(client=None, table_names=[]): day, api_key """ ) + if "daily_aggregated_spend_per_user_mv" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_user_mv") + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_per_user_mv + TO daily_aggregated_spend_per_user + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend, + user as user + FROM spend_logs + GROUP BY + day, user + """ + ) if "daily_aggregated_spend_mv" not in table_names: verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_mv") client.command( @@ -116,6 +132,20 @@ def _create_clickhouse_aggregate_tables(client=None, table_names=[]): ORDER BY (day, api_key); """ ) + if "daily_aggregated_spend_per_user" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_user") + client.command( + """ + CREATE TABLE daily_aggregated_spend_per_user + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + `user` String + ) + ENGINE = SummingMergeTree() + ORDER BY (day, user); + """ + ) if "daily_aggregated_spend" not in table_names: verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend") client.command( From 57998c28dc9a3af079e6efa1b15f2f6348a7afd9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:37:29 -0800 Subject: [PATCH 03/26] fix(proxy_server.py): drop none values in streaming response --- litellm/proxy/proxy_server.py | 5 ++--- litellm/utils.py | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 52f2783a8..bc6f8ccc5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2115,10 +2115,9 @@ async def async_data_generator(response, user_api_key_dict): try: start_time = time.time() async for chunk in response: - verbose_proxy_logger.debug(f"returned chunk: {chunk}") - assert isinstance(chunk, litellm.ModelResponse) + chunk = chunk.model_dump_json(exclude_none=True) try: - yield f"data: {json.dumps(chunk.model_dump(exclude_none=True))}\n\n" + yield f"data: {chunk}\n\n" except Exception as e: yield f"data: {str(e)}\n\n" diff --git a/litellm/utils.py b/litellm/utils.py index 6bc229e96..d8986be6b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -205,18 +205,18 @@ def map_finish_reason( class FunctionCall(OpenAIObject): arguments: str - name: str + name: Optional[str] = None class Function(OpenAIObject): arguments: str - name: str + name: Optional[str] = None class ChatCompletionDeltaToolCall(OpenAIObject): - id: str + id: Optional[str] = None function: Function - type: str + type: Optional[str] = None index: int @@ -275,8 +275,11 @@ class Delta(OpenAIObject): super(Delta, self).__init__(**params) self.content = content self.role = role - self.function_call = function_call - if tool_calls is not None and isinstance(tool_calls, dict): + if function_call is not None and isinstance(function_call, dict): + self.function_call = FunctionCall(**function_call) + else: + self.function_call = function_call + if tool_calls is not None and isinstance(tool_calls, list): self.tool_calls = [] for tool_call in tool_calls: if tool_call.get("index", None) is None: @@ -8727,7 +8730,7 @@ class CustomStreamWrapper: or original_chunk.choices[0].delta.tool_calls is not None ): try: - delta = dict(original_chunk.choices[0].delta) + delta = original_chunk.choices[0].delta model_response.system_fingerprint = ( original_chunk.system_fingerprint ) @@ -8762,7 +8765,9 @@ class CustomStreamWrapper: is None ): t.function.arguments = "" - model_response.choices[0].delta = Delta(**delta) + _json_delta = delta.model_dump() + print_verbose(f"_json_delta: {_json_delta}") + model_response.choices[0].delta = Delta(**_json_delta) except Exception as e: traceback.print_exc() model_response.choices[0].delta = Delta() From f3a6225f6110321a919bfc22d3f1f33a8a594593 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:38:40 -0800 Subject: [PATCH 04/26] =?UTF-8?q?bump:=20version=201.27.11=20=E2=86=92=201?= =?UTF-8?q?.27.12.dev1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e914e6941..bfe45dbaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.27.11" +version = "1.27.12.dev1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -74,7 +74,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.27.11" +version = "1.27.12.dev1" version_files = [ "pyproject.toml:^version" ] From 66d552eb9a53dc39afc23d9bb316b060fbedf529 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 14:51:23 -0800 Subject: [PATCH 05/26] (feat) /daily/metrics --- litellm/integrations/clickhouse.py | 120 +++++++++++++++++++++++++++++ litellm/proxy/proxy_server.py | 56 ++++++++++++++ 2 files changed, 176 insertions(+) diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 82ecd3256..700423664 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -27,6 +27,126 @@ import litellm, uuid from litellm._logging import print_verbose, verbose_logger +def create_client(): + try: + import clickhouse_connect + + port = os.getenv("CLICKHOUSE_PORT") + clickhouse_host = os.getenv("CLICKHOUSE_HOST") + if clickhouse_host is not None: + verbose_logger.debug("setting up clickhouse") + if port is not None and isinstance(port, str): + port = int(port) + + client = clickhouse_connect.get_client( + host=os.getenv("CLICKHOUSE_HOST"), + port=port, + username=os.getenv("CLICKHOUSE_USERNAME"), + password=os.getenv("CLICKHOUSE_PASSWORD"), + ) + return client + else: + raise Exception("Clickhouse: Clickhouse host not set") + except Exception as e: + raise ValueError(f"Clickhouse: {e}") + + +def build_daily_metrics(): + click_house_client = create_client() + + # get daily spend + daily_spend = click_house_client.query_df( + """ + SELECT sumMerge(DailySpend) as daily_spend, day FROM daily_aggregated_spend GROUP BY day + """ + ) + + # get daily spend per model + + daily_spend_per_model = click_house_client.query_df( + """ + SELECT sumMerge(DailySpend) as daily_spend, day, model FROM daily_aggregated_spend_per_model GROUP BY day, model + """ + ) + new_df = daily_spend_per_model.to_dict(orient="records") + import pandas as pd + + df = pd.DataFrame(new_df) + # Group by 'day' and create a dictionary for each group + result_dict = {} + for day, group in df.groupby("day"): + models = group["model"].tolist() + spend = group["daily_spend"].tolist() + spend_per_model = {model: spend for model, spend in zip(models, spend)} + result_dict[day] = spend_per_model + + # Display the resulting dictionary + + daily_spend_per_api_key = click_house_client.query_df( + """ + SELECT + daily_spend, + day, + api_key + FROM ( + SELECT + sumMerge(DailySpend) as daily_spend, + day, + api_key, + RANK() OVER (PARTITION BY day ORDER BY sumMerge(DailySpend) DESC) as spend_rank + FROM + daily_aggregated_spend_per_api_key + GROUP BY + day, + api_key + ) AS ranked_api_keys + WHERE + spend_rank <= 5 + AND day IS NOT NULL + ORDER BY + day, + daily_spend DESC + """ + ) + new_df = daily_spend_per_api_key.to_dict(orient="records") + import pandas as pd + + df = pd.DataFrame(new_df) + # Group by 'day' and create a dictionary for each group + api_key_result_dict = {} + for day, group in df.groupby("day"): + api_keys = group["api_key"].tolist() + spend = group["daily_spend"].tolist() + spend_per_api_key = {api_key: spend for api_key, spend in zip(api_keys, spend)} + api_key_result_dict[day] = spend_per_api_key + + # Display the resulting dictionary + + # for each day in daily spend, look up the day in result_dict and api_key_result_dict + # Assuming daily_spend DataFrame has 'day' column + result = [] + for index, row in daily_spend.iterrows(): + day = row["day"] + data_day = row.to_dict() + + # Look up in result_dict + if day in result_dict: + spend_per_model = result_dict[day] + # Assuming there is a column named 'model' in daily_spend + data_day["spend_per_model"] = spend_per_model # Assign 0 if model not found + + # Look up in api_key_result_dict + if day in api_key_result_dict: + spend_per_api_key = api_key_result_dict[day] + # Assuming there is a column named 'api_key' in daily_spend + data_day["spend_per_api_key"] = spend_per_api_key + + result.append(data_day) + + # print("FINAL daily metric", result) + return result + + def _create_clickhouse_material_views(client=None, table_names=[]): # Create Materialized Views if they don't exist # Materialized Views send new inserted rows to the aggregate tables diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6b9e3f644..d4b261817 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3991,6 +3991,62 @@ async def view_spend_logs( ) +@router.get( + "/daily_metrics", + summary="Get daily spend metrics", + tags=["budget & spend Tracking"], + dependencies=[Depends(user_api_key_auth)], +) +async def view_daily_metrics( + start_date: Optional[str] = fastapi.Query( + default=None, + description="Time from which to start viewing key spend", + ), + end_date: Optional[str] = fastapi.Query( + default=None, + description="Time till which to view key spend", + ), + metric_type: Optional[Literal["per_api_key", "per_user", "per_model"]] = None, +): + """ """ + try: + if os.getenv("CLICKHOUSE_HOST") is not None: + # gettting spend logs from clickhouse + from litellm.integrations import clickhouse + + return clickhouse.build_daily_metrics() + + # create a response object + """ + { + "date": "2022-01-01", + "spend": 0.0, + "users": {}, + "models": {} + } + """ + else: + raise Exception( + "Clickhouse: Clickhouse host not set. Required for viewing /daily/metrics" + ) + except Exception as e: + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"/spend/logs Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="/spend/logs Error" + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + #### USER MANAGEMENT #### @router.post( "/user/new", From 39d8645596c54e5c1495714f56ae2e7f546e1c4f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:53:46 -0800 Subject: [PATCH 06/26] test(test_proxy_custom_logger.py): fix format test --- litellm/tests/test_proxy_custom_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_proxy_custom_logger.py b/litellm/tests/test_proxy_custom_logger.py index 34e427ef4..64ed08897 100644 --- a/litellm/tests/test_proxy_custom_logger.py +++ b/litellm/tests/test_proxy_custom_logger.py @@ -274,7 +274,7 @@ def test_chat_completion_stream(client): print("\n\n decode_data", data) # Access the content of choices[0]['message']['content'] - content = data["choices"][0]["delta"]["content"] or "" + content = data["choices"][0]["delta"].get("content", None) or "" # Process the content as needed print("Content:", content) From 94f4f969943ca8304b09661c86d7086a681fabb0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 14:57:50 -0800 Subject: [PATCH 07/26] fix(utils.py): fix streaming issue --- litellm/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index d8986be6b..658ebba65 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -282,9 +282,12 @@ class Delta(OpenAIObject): if tool_calls is not None and isinstance(tool_calls, list): self.tool_calls = [] for tool_call in tool_calls: - if tool_call.get("index", None) is None: - tool_call["index"] = 0 - self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call)) + if isinstance(tool_call, dict): + if tool_call.get("index", None) is None: + tool_call["index"] = 0 + self.tool_calls.append(ChatCompletionDeltaToolCall(**tool_call)) + elif isinstance(tool_call, ChatCompletionDeltaToolCall): + self.tool_calls.append(tool_call) else: self.tool_calls = tool_calls From e7b88c2134a013f527304de29358238a5593f91f Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 27 Feb 2024 16:47:09 -0800 Subject: [PATCH 08/26] Update model_prices_and_context_window.json --- model_prices_and_context_window.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6c663200d..5bd0bcdff 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -687,6 +687,15 @@ "litellm_provider": "vertex_ai-language-models", "mode": "chat" }, + "gemini-1.5-pro-preview-0215": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat" + }, "gemini-pro-vision": { "max_tokens": 16384, "max_output_tokens": 2048, From f3144dd9cf790eda0c1ee070c01af087668e589a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 17:44:35 -0800 Subject: [PATCH 09/26] (docs) vertex ai --- docs/my-website/docs/providers/vertex.md | 19 +++++++++++-------- ...odel_prices_and_context_window_backup.json | 9 +++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 70ee9eca9..20225473b 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -25,14 +25,17 @@ response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "co 1. Modify the config.yaml ```yaml -litellm_settings: - vertex_project: "hardy-device-38811" # Your Project ID - vertex_location: "us-central1" # proj location - -model_list: - -model_name: team1-gemini-pro - litellm_params: - model: gemini-pro +model_list: + - model_name: gemini-vision + litellm_params: + model: vertex_ai/gemini-1.0-pro-vision-001 + vertex_project: "project-id" + vertex_location: "us-central1" + - model_name: gemini-vision + litellm_params: + model: vertex_ai/gemini-1.0-pro-vision-001 + vertex_project: "project-id2" + vertex_location: "us-east" ``` 2. Start the proxy diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6c663200d..5bd0bcdff 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -687,6 +687,15 @@ "litellm_provider": "vertex_ai-language-models", "mode": "chat" }, + "gemini-1.5-pro-preview-0215": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat" + }, "gemini-pro-vision": { "max_tokens": 16384, "max_output_tokens": 2048, From 7485fa797c7d2eee715eebbcc7de2e7cf7a8b265 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 17:55:05 -0800 Subject: [PATCH 10/26] (docs) vertex ai litellm proxy --- docs/my-website/docs/providers/vertex.md | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 20225473b..d959498ce 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # VertexAI - Google [Gemini, Model Garden] @@ -22,8 +25,16 @@ response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "co ## OpenAI Proxy Usage +Here's how to use Vertex AI with the LiteLLM Proxy Server + 1. Modify the config.yaml + + + + +Use this when you need to set a different location for each vertex model + ```yaml model_list: - model_name: gemini-vision @@ -38,6 +49,27 @@ model_list: vertex_location: "us-east" ``` + + + + +Use this when you have one vertex location for all models + +```yaml +litellm_settings: + vertex_project: "hardy-device-38811" # Your Project ID + vertex_location: "us-central1" # proj location + +model_list: + -model_name: team1-gemini-pro + litellm_params: + model: gemini-pro +``` + + + + + 2. Start the proxy ```bash From 7cabc6ac565c0b03392daedd2a81d2f4482396e9 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 18:03:15 -0800 Subject: [PATCH 11/26] (docs) show daily metrics tab --- docs/my-website/docs/proxy/metrics.md | 32 +++++++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 33 insertions(+) create mode 100644 docs/my-website/docs/proxy/metrics.md diff --git a/docs/my-website/docs/proxy/metrics.md b/docs/my-website/docs/proxy/metrics.md new file mode 100644 index 000000000..d113ff6fb --- /dev/null +++ b/docs/my-website/docs/proxy/metrics.md @@ -0,0 +1,32 @@ +# 💸 GET Daily Spend, Usage Metrics + +## Response format +```json +[ + { + "daily_spend": 7.9261938052047e+16, + "day": "2024-02-01T00:00:00", + "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16}, + "spend_per_api_key": { + "76": 914495704992000.0, + "12": 905726697912000.0, + "71": 866312628003000.0, + "28": 865461799332000.0, + "13": 859151538396000.0 + } + }, + { + "daily_spend": 7.938489251309491e+16, + "day": "2024-02-02T00:00:00", + "spend_per_model": {"gpt-3.5": 7.938489251309491e+16}, + "spend_per_api_key": { + "91": 896805036036000.0, + "78": 889692646082000.0, + "49": 885386687861000.0, + "28": 873869890984000.0, + "56": 867398637692000.0 + } + } +] + +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 7a24723af..d69abcbfb 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -40,6 +40,7 @@ const sidebars = { "proxy/virtual_keys", "proxy/users", "proxy/ui", + "proxy/metrics", "proxy/model_management", "proxy/health", "proxy/debugging", From d19370083d711aa607b15284aaac70d06dfbab70 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 18:33:59 -0800 Subject: [PATCH 12/26] (feat) update /daily metrics --- docs/my-website/docs/proxy/metrics.md | 53 +++++++++++++++------------ litellm/integrations/clickhouse.py | 31 ++++++++++++++-- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/docs/my-website/docs/proxy/metrics.md b/docs/my-website/docs/proxy/metrics.md index d113ff6fb..5b839dbd4 100644 --- a/docs/my-website/docs/proxy/metrics.md +++ b/docs/my-website/docs/proxy/metrics.md @@ -3,30 +3,37 @@ ## Response format ```json [ - { - "daily_spend": 7.9261938052047e+16, - "day": "2024-02-01T00:00:00", - "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16}, - "spend_per_api_key": { - "76": 914495704992000.0, - "12": 905726697912000.0, - "71": 866312628003000.0, - "28": 865461799332000.0, - "13": 859151538396000.0 + daily_spend = [ + { + "daily_spend": 7.9261938052047e+16, + "day": "2024-02-01T00:00:00", + "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16}, + "spend_per_api_key": { + "76": 914495704992000.0, + "12": 905726697912000.0, + "71": 866312628003000.0, + "28": 865461799332000.0, + "13": 859151538396000.0 + } + }, + { + "daily_spend": 7.938489251309491e+16, + "day": "2024-02-02T00:00:00", + "spend_per_model": {"gpt-3.5": 7.938489251309491e+16}, + "spend_per_api_key": { + "91": 896805036036000.0, + "78": 889692646082000.0, + "49": 885386687861000.0, + "28": 873869890984000.0, + "56": 867398637692000.0 + } } - }, - { - "daily_spend": 7.938489251309491e+16, - "day": "2024-02-02T00:00:00", - "spend_per_model": {"gpt-3.5": 7.938489251309491e+16}, - "spend_per_api_key": { - "91": 896805036036000.0, - "78": 889692646082000.0, - "49": 885386687861000.0, - "28": 873869890984000.0, - "56": 867398637692000.0 - } - } + + ], + total_spend = 200, + top_models = {"gpt4": 0.2, "vertexai/gemini-pro":10}, + top_api_keys = {"899922": 0.9, "838hcjd999seerr88": 20} + ] ``` \ No newline at end of file diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 700423664..55eaaa090 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -62,7 +62,6 @@ def build_daily_metrics(): ) # get daily spend per model - daily_spend_per_model = click_house_client.query_df( """ SELECT sumMerge(DailySpend) as daily_spend, day, model FROM daily_aggregated_spend_per_model GROUP BY day, model @@ -82,6 +81,7 @@ def build_daily_metrics(): # Display the resulting dictionary + # get daily spend per API key daily_spend_per_api_key = click_house_client.query_df( """ SELECT @@ -122,6 +122,23 @@ def build_daily_metrics(): # Display the resulting dictionary + # Calculate total spend across all days + total_spend = daily_spend["daily_spend"].sum() + + # Identify top models and top API keys with the highest spend across all days + top_models = {} + top_api_keys = {} + + for day, spend_per_model in result_dict.items(): + for model, model_spend in spend_per_model.items(): + if model not in top_models or model_spend > top_models[model]: + top_models[model] = model_spend + + for day, spend_per_api_key in api_key_result_dict.items(): + for api_key, api_key_spend in spend_per_api_key.items(): + if api_key not in top_api_keys or api_key_spend > top_api_keys[api_key]: + top_api_keys[api_key] = api_key_spend + # for each day in daily spend, look up the day in result_dict and api_key_result_dict # Assuming daily_spend DataFrame has 'day' column result = [] @@ -143,8 +160,16 @@ def build_daily_metrics(): result.append(data_day) - # print("FINAL daily metric", result) - return result + data_to_return = {} + data_to_return["daily_spend"] = result + + data_to_return["total_spend"] = total_spend + data_to_return["top_models"] = top_models + data_to_return["top_api_keys"] = top_api_keys + return data_to_return + + +# build_daily_metrics() def _create_clickhouse_material_views(client=None, table_names=[]): From 6352c8869cbb2522aff0df5f5177a72070c0c5e5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 18:34:07 -0800 Subject: [PATCH 13/26] refactor(vertex_ai.py): add more logging to show the vertex ai client init params --- litellm/llms/vertex_ai.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/llms/vertex_ai.py b/litellm/llms/vertex_ai.py index f4447a9e9..18c06d4d6 100644 --- a/litellm/llms/vertex_ai.py +++ b/litellm/llms/vertex_ai.py @@ -278,7 +278,11 @@ def completion( import google.auth ## Load credentials with the correct quota project ref: https://github.com/googleapis/python-aiplatform/issues/2557#issuecomment-1709284744 + print_verbose( + f"VERTEX AI: vertex_project={vertex_project}; vertex_location={vertex_location}" + ) creds, _ = google.auth.default(quota_project_id=vertex_project) + print_verbose(f"VERTEX AI: creds={creds}") vertexai.init( project=vertex_project, location=vertex_location, credentials=creds ) From 8c5cdb6744eb5a1bfede9d0738fdea3580f78269 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 18:36:16 -0800 Subject: [PATCH 14/26] (feat) update /daily metrics --- litellm/proxy/proxy_server.py | 49 ++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d4b261817..b00e7f231 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3832,13 +3832,51 @@ async def view_spend_logs( # gettting spend logs from clickhouse from litellm.proxy.enterprise.utils import view_spend_logs_from_clickhouse - return await view_spend_logs_from_clickhouse( - api_key=api_key, - user_id=user_id, - request_id=request_id, + daily_metrics = await view_daily_metrics( start_date=start_date, end_date=end_date, ) + + # get the top api keys across all daily_metrics + top_api_keys = {} # type: ignore + + # make this compatible with the admin UI + for response in daily_metrics.get("daily_spend", {}): + response["startTime"] = response["day"] + response["spend"] = response["daily_spend"] + response["models"] = response["spend_per_model"] + response["users"] = {"ishaan": 0.0} + spend_per_api_key = response["spend_per_api_key"] + + # insert spend_per_api_key key, values in response + for key, value in spend_per_api_key.items(): + response[key] = value + top_api_keys[key] = top_api_keys.get(key, 0.0) + value + + del response["day"] + del response["daily_spend"] + del response["spend_per_model"] + del response["spend_per_api_key"] + + # get top 5 api keys + top_api_keys = sorted(top_api_keys.items(), key=lambda x: x[1], reverse=True) # type: ignore + top_api_keys = top_api_keys[:5] # type: ignore + top_api_keys = dict(top_api_keys) # type: ignore + """ + set it like this + { + "key" : key, + "spend:" : spend + } + """ + response_keys = {} + for key in top_api_keys.items(): + response_keys["key"] = key[0] + response_keys["spend"] = key[1] + + daily_metrics["top_api_keys"] = response_keys + + return daily_metrics global prisma_client try: verbose_proxy_logger.debug("inside view_spend_logs") @@ -4006,7 +4044,6 @@ async def view_daily_metrics( default=None, description="Time till which to view key spend", ), - metric_type: Optional[Literal["per_api_key", "per_user", "per_model"]] = None, ): """ """ try: @@ -4022,7 +4059,7 @@ async def view_daily_metrics( "date": "2022-01-01", "spend": 0.0, "users": {}, - "models": {} + "models": {}, } """ else: From 4fb723f119b5836e96ceea1f550eb618d271faf3 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:34:47 -0800 Subject: [PATCH 15/26] (docs) show /daily_metrics --- docs/my-website/docs/proxy/metrics.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/my-website/docs/proxy/metrics.md b/docs/my-website/docs/proxy/metrics.md index 5b839dbd4..bf5ebe285 100644 --- a/docs/my-website/docs/proxy/metrics.md +++ b/docs/my-website/docs/proxy/metrics.md @@ -1,5 +1,10 @@ # 💸 GET Daily Spend, Usage Metrics +## Request Format +```shell +curl -X GET "http://0.0.0.0:4000/daily_metrics" -H "Authorization: Bearer sk-1234" +``` + ## Response format ```json [ From e6d13261ce83aae1ac09cb619a3e7ed2204c597f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:35:18 -0800 Subject: [PATCH 16/26] (feat) show analytics on admin UI --- litellm/proxy/proxy_server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b00e7f231..f174f6685 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3869,11 +3869,15 @@ async def view_spend_logs( "spend:" : spend } """ - response_keys = {} + # we need this to show on the Admin UI + response_keys = [] for key in top_api_keys.items(): - response_keys["key"] = key[0] - response_keys["spend"] = key[1] - + response_keys.append( + { + "key": key[0], + "spend": key[1], + } + ) daily_metrics["top_api_keys"] = response_keys return daily_metrics From 02dc031bb599896df91324ba60941907d5f2ae95 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:35:40 -0800 Subject: [PATCH 17/26] (ui) show usage on ui --- ui/litellm-dashboard/src/components/usage.tsx | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 84c942eec..53493eabc 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -172,20 +172,32 @@ const UsagePage: React.FC = ({ startTime, endTime ).then(async (response) => { - const topKeysResponse = await keyInfoCall( - accessToken, - getTopKeys(response) - ); - const filtered_keys = topKeysResponse["info"].map((k: any) => ({ - key: (k["key_name"] || k["key_alias"] || k["token"]).substring( - 0, - 7 - ), - spend: k["spend"], - })); - setTopKeys(filtered_keys); - setTopUsers(getTopUsers(response)); - setKeySpendData(response); + console.log("result from spend logs call", response); + if ("daily_spend" in response) { + // this is from clickhouse analytics + // + let daily_spend = response["daily_spend"]; + console.log("daily spend", daily_spend); + setKeySpendData(daily_spend); + let topApiKeys = response.top_api_keys; + setTopKeys(topApiKeys); + } + else { + const topKeysResponse = await keyInfoCall( + accessToken, + getTopKeys(response) + ); + const filtered_keys = topKeysResponse["info"].map((k: any) => ({ + key: (k["key_name"] || k["key_alias"] || k["token"]).substring( + 0, + 7 + ), + spend: k["spend"], + })); + setTopKeys(filtered_keys); + setTopUsers(getTopUsers(response)); + setKeySpendData(response); + } }); } catch (error) { console.error("There was an error fetching the data", error); From d9a3c4179f50020cfc60b18109c0e62c395a6456 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:38:18 -0800 Subject: [PATCH 18/26] (feat) pin pandas on req.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6bd69302f..caede5b67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ gunicorn==21.2.0 # server dep boto3==1.34.34 # aws bedrock/sagemaker calls redis==5.0.0 # caching numpy==1.24.3 # semantic caching +pandas==2.1.1 # for viewing clickhouse spend analytics prisma==0.11.0 # for db mangum==0.17.0 # for aws lambda functions google-generativeai==0.3.2 # for vertex ai calls From 9ad8b7c8fb003662c5c95549f97e7edaae94ee2c Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:41:01 -0800 Subject: [PATCH 19/26] (chore) move clickhouse to correct directory --- enterprise/utils.py | 135 ++++++++++++++++++++++++++++ litellm/integrations/clickhouse.py | 138 +---------------------------- 2 files changed, 138 insertions(+), 135 deletions(-) diff --git a/enterprise/utils.py b/enterprise/utils.py index f4916b689..3b5a90fc0 100644 --- a/enterprise/utils.py +++ b/enterprise/utils.py @@ -110,3 +110,138 @@ async def view_spend_logs_from_clickhouse( "log_count": num_rows, } return response_data + + +def _create_clickhouse_material_views(client=None, table_names=[]): + # Create Materialized Views if they don't exist + # Materialized Views send new inserted rows to the aggregate tables + + verbose_logger.debug("Clickhouse: Creating Materialized Views") + if "daily_aggregated_spend_per_model_mv" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_model_mv") + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_per_model_mv + TO daily_aggregated_spend_per_model + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend, + model as model + FROM spend_logs + GROUP BY + day, model + """ + ) + if "daily_aggregated_spend_per_api_key_mv" not in table_names: + verbose_logger.debug( + "Clickhouse: Creating daily_aggregated_spend_per_api_key_mv" + ) + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_per_api_key_mv + TO daily_aggregated_spend_per_api_key + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend, + api_key as api_key + FROM spend_logs + GROUP BY + day, api_key + """ + ) + if "daily_aggregated_spend_per_user_mv" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_user_mv") + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_per_user_mv + TO daily_aggregated_spend_per_user + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend, + user as user + FROM spend_logs + GROUP BY + day, user + """ + ) + if "daily_aggregated_spend_mv" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_mv") + client.command( + """ + CREATE MATERIALIZED VIEW daily_aggregated_spend_mv + TO daily_aggregated_spend + AS + SELECT + toDate(startTime) as day, + sumState(spend) AS DailySpend + FROM spend_logs + GROUP BY + day + """ + ) + + +def _create_clickhouse_aggregate_tables(client=None, table_names=[]): + # Basic Logging works without this - this is only used for low latency reporting apis + verbose_logger.debug("Clickhouse: Creating Aggregate Tables") + + # Create Aggregeate Tables if they don't exist + if "daily_aggregated_spend_per_model" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_model") + client.command( + """ + CREATE TABLE daily_aggregated_spend_per_model + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + `model` String + ) + ENGINE = SummingMergeTree() + ORDER BY (day, model); + """ + ) + if "daily_aggregated_spend_per_api_key" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_api_key") + client.command( + """ + CREATE TABLE daily_aggregated_spend_per_api_key + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + `api_key` String + ) + ENGINE = SummingMergeTree() + ORDER BY (day, api_key); + """ + ) + if "daily_aggregated_spend_per_user" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_user") + client.command( + """ + CREATE TABLE daily_aggregated_spend_per_user + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + `user` String + ) + ENGINE = SummingMergeTree() + ORDER BY (day, user); + """ + ) + if "daily_aggregated_spend" not in table_names: + verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend") + client.command( + """ + CREATE TABLE daily_aggregated_spend + ( + `day` Date, + `DailySpend` AggregateFunction(sum, Float64), + ) + ENGINE = SummingMergeTree() + ORDER BY (day); + """ + ) + return diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 82ecd3256..a2a0b800e 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -27,141 +27,6 @@ import litellm, uuid from litellm._logging import print_verbose, verbose_logger -def _create_clickhouse_material_views(client=None, table_names=[]): - # Create Materialized Views if they don't exist - # Materialized Views send new inserted rows to the aggregate tables - - verbose_logger.debug("Clickhouse: Creating Materialized Views") - if "daily_aggregated_spend_per_model_mv" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_model_mv") - client.command( - """ - CREATE MATERIALIZED VIEW daily_aggregated_spend_per_model_mv - TO daily_aggregated_spend_per_model - AS - SELECT - toDate(startTime) as day, - sumState(spend) AS DailySpend, - model as model - FROM spend_logs - GROUP BY - day, model - """ - ) - if "daily_aggregated_spend_per_api_key_mv" not in table_names: - verbose_logger.debug( - "Clickhouse: Creating daily_aggregated_spend_per_api_key_mv" - ) - client.command( - """ - CREATE MATERIALIZED VIEW daily_aggregated_spend_per_api_key_mv - TO daily_aggregated_spend_per_api_key - AS - SELECT - toDate(startTime) as day, - sumState(spend) AS DailySpend, - api_key as api_key - FROM spend_logs - GROUP BY - day, api_key - """ - ) - if "daily_aggregated_spend_per_user_mv" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_user_mv") - client.command( - """ - CREATE MATERIALIZED VIEW daily_aggregated_spend_per_user_mv - TO daily_aggregated_spend_per_user - AS - SELECT - toDate(startTime) as day, - sumState(spend) AS DailySpend, - user as user - FROM spend_logs - GROUP BY - day, user - """ - ) - if "daily_aggregated_spend_mv" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_mv") - client.command( - """ - CREATE MATERIALIZED VIEW daily_aggregated_spend_mv - TO daily_aggregated_spend - AS - SELECT - toDate(startTime) as day, - sumState(spend) AS DailySpend - FROM spend_logs - GROUP BY - day - """ - ) - - -def _create_clickhouse_aggregate_tables(client=None, table_names=[]): - # Basic Logging works without this - this is only used for low latency reporting apis - verbose_logger.debug("Clickhouse: Creating Aggregate Tables") - - # Create Aggregeate Tables if they don't exist - if "daily_aggregated_spend_per_model" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_model") - client.command( - """ - CREATE TABLE daily_aggregated_spend_per_model - ( - `day` Date, - `DailySpend` AggregateFunction(sum, Float64), - `model` String - ) - ENGINE = SummingMergeTree() - ORDER BY (day, model); - """ - ) - if "daily_aggregated_spend_per_api_key" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_api_key") - client.command( - """ - CREATE TABLE daily_aggregated_spend_per_api_key - ( - `day` Date, - `DailySpend` AggregateFunction(sum, Float64), - `api_key` String - ) - ENGINE = SummingMergeTree() - ORDER BY (day, api_key); - """ - ) - if "daily_aggregated_spend_per_user" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend_per_user") - client.command( - """ - CREATE TABLE daily_aggregated_spend_per_user - ( - `day` Date, - `DailySpend` AggregateFunction(sum, Float64), - `user` String - ) - ENGINE = SummingMergeTree() - ORDER BY (day, user); - """ - ) - if "daily_aggregated_spend" not in table_names: - verbose_logger.debug("Clickhouse: Creating daily_aggregated_spend") - client.command( - """ - CREATE TABLE daily_aggregated_spend - ( - `day` Date, - `DailySpend` AggregateFunction(sum, Float64), - ) - ENGINE = SummingMergeTree() - ORDER BY (day); - """ - ) - return - - def _start_clickhouse(): import clickhouse_connect @@ -223,6 +88,9 @@ def _start_clickhouse(): # RUN Enterprise Clickhouse Setup # TLDR: For Enterprise - we create views / aggregate tables for low latency reporting APIs + from litellm.proxy.enterprise.utils import _create_clickhouse_aggregate_tables + from litellm.proxy.enterprise.utils import _create_clickhouse_material_views + _create_clickhouse_aggregate_tables(client=client, table_names=table_names) _create_clickhouse_material_views(client=client, table_names=table_names) From 6f60acefa5df625000a672066fd4830e89a3d94b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 19:50:40 -0800 Subject: [PATCH 20/26] build(config.yml): install fastapi during docker build test --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a24ae1d8e..daa4d59ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,6 +130,7 @@ jobs: pip install "langfuse>=2.0.0" pip install numpydoc pip install prisma + pip install fastapi pip install "httpx==0.24.1" pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" From 3af6526aa6564930255662861b421786c98ac041 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 20:34:15 -0800 Subject: [PATCH 21/26] =?UTF-8?q?bump:=20version=201.27.12=20=E2=86=92=201?= =?UTF-8?q?.27.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 295d7e902..6265c2dde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.27.12" +version = "1.27.13" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -74,7 +74,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.27.12" +version = "1.27.13" version_files = [ "pyproject.toml:^version" ] From a6254af1f1c2ae23b14a6f14a03b701037a0d911 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 20:35:06 -0800 Subject: [PATCH 22/26] (ci/cd) run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 71f77c9b4..143f730dd 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -97,7 +97,7 @@ def test_completion_mistral_api(): } ], ) - # Add any assertions here to check the response + # Add any assertions here, to check the response print(response) cost = litellm.completion_cost(completion_response=response) From 2fbcd0e07368601779043effc1655fd395898ac3 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 20:35:21 -0800 Subject: [PATCH 23/26] =?UTF-8?q?bump:=20version=201.27.13=20=E2=86=92=201?= =?UTF-8?q?.27.14?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6265c2dde..ce452a2d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.27.13" +version = "1.27.14" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -74,7 +74,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.27.13" +version = "1.27.14" version_files = [ "pyproject.toml:^version" ] From 5265d9042d1c445fd729b691a9a0a9f6eee9271d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 20:35:32 -0800 Subject: [PATCH 24/26] test: testing fixes --- litellm/proxy/proxy_server.py | 15 ++++++++++++--- litellm/tests/test_amazing_vertex_completion.py | 4 ++++ proxy_server_config.yaml | 2 ++ tests/test_keys.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 86c7e7ecf..5ce526d50 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -239,6 +239,8 @@ health_check_interval = None health_check_results = {} queue: List = [] litellm_proxy_budget_name = "litellm-proxy-budget" +proxy_budget_rescheduler_min_time = 597 +proxy_budget_rescheduler_max_time = 605 ### INITIALIZE GLOBAL LOGGING OBJECT ### proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) ### REDIS QUEUE ### @@ -1406,7 +1408,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, use_background_health_checks, health_check_interval, use_queue, custom_db_client + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, use_background_health_checks, health_check_interval, use_queue, custom_db_client, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time # Load existing config config = await self.get_config(config_file_path=config_file_path) @@ -1713,6 +1715,13 @@ class ProxyConfig: ) ## COST TRACKING ## cost_tracking() + ## BUDGET RESCHEDULER ## + proxy_budget_rescheduler_min_time = general_settings.get( + "proxy_budget_rescheduler_min_time", proxy_budget_rescheduler_min_time + ) + proxy_budget_rescheduler_max_time = general_settings.get( + "proxy_budget_rescheduler_max_time", proxy_budget_rescheduler_max_time + ) ### BACKGROUND HEALTH CHECKS ### # Enable background health checks use_background_health_checks = general_settings.get( @@ -2196,7 +2205,7 @@ def parse_cache_control(cache_control): @router.on_event("startup") async def startup_event(): - global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings + global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time import json ### LOAD MASTER KEY ### @@ -2307,7 +2316,7 @@ async def startup_event(): if prisma_client is not None: scheduler = AsyncIOScheduler() interval = random.randint( - 597, 605 + proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time ) # random interval, so multiple workers avoid resetting budget at the same time scheduler.add_job( reset_budget, "interval", seconds=interval, args=[prisma_client] diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index fb2ebc0d6..1de26d3b9 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -110,6 +110,7 @@ def test_vertex_ai(): "code-bison@001", "text-bison@001", "gemini-1.5-pro", + "gemini-1.5-pro-preview-0215", "gemini-1.5-pro-vision", ]: # our account does not have access to this model @@ -160,6 +161,7 @@ def test_vertex_ai_stream(): "code-bison@001", "text-bison@001", "gemini-1.5-pro", + "gemini-1.5-pro-preview-0215", "gemini-1.5-pro-vision", ]: # our account does not have access to this model @@ -211,6 +213,7 @@ async def test_async_vertexai_response(): "code-bison@001", "text-bison@001", "gemini-1.5-pro", + "gemini-1.5-pro-preview-0215", "gemini-1.5-pro-vision", ]: # our account does not have access to this model @@ -255,6 +258,7 @@ async def test_async_vertexai_streaming_response(): "code-bison@001", "text-bison@001", "gemini-1.5-pro", + "gemini-1.5-pro-preview-0215", "gemini-1.5-pro-vision", ]: # our account does not have access to this model diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index d0cb5739e..198d33013 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -40,6 +40,8 @@ litellm_settings: budget_duration: 30d general_settings: master_key: sk-1234 # [OPTIONAL] Only use this if you to require all calls to contain this key (Authorization: Bearer sk-1234) + proxy_budget_rescheduler_min_time: 30 + proxy_budget_rescheduler_max_time: 60 # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy environment_variables: diff --git a/tests/test_keys.py b/tests/test_keys.py index 28ce02511..c2b957180 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -449,7 +449,7 @@ async def test_key_with_budgets(): reset_at_init_value = key_info["info"]["budget_reset_at"] reset_at_new_value = None i = 0 - await asyncio.sleep(610) + await asyncio.sleep(120) while i < 3: key_info = await get_key_info(session=session, get_key=key, call_key=key) reset_at_new_value = key_info["info"]["budget_reset_at"] From a40590d849e8fefe61aa557e9d1c3d2d0bed7a09 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 20:49:04 -0800 Subject: [PATCH 25/26] (ci/cd) run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 143f730dd..71f77c9b4 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -97,7 +97,7 @@ def test_completion_mistral_api(): } ], ) - # Add any assertions here, to check the response + # Add any assertions here to check the response print(response) cost = litellm.completion_cost(completion_response=response) From 4f3e8d06272d04e1b97df1e39fc5e9b2150645e0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 27 Feb 2024 20:58:45 -0800 Subject: [PATCH 26/26] test(test_amazing_vertex_completion.py): fix test --- litellm/tests/test_amazing_vertex_completion.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 1de26d3b9..f5df00c8d 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -130,6 +130,8 @@ def test_vertex_ai(): f"response.choices[0].finish_reason: {response.choices[0].finish_reason}" ) assert response.choices[0].finish_reason in litellm._openai_finish_reasons + except litellm.RateLimitError as e: + pass except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -183,6 +185,8 @@ def test_vertex_ai_stream(): assert type(content) == str # pass assert len(completed_str) > 4 + except litellm.RateLimitError as e: + pass except Exception as e: pytest.fail(f"Error occurred: {e}")