From d384a0b39fbc5e8f805aa87e43a8feb78e325f6c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 09:25:31 -0700 Subject: [PATCH 01/11] feat - send daily spend reports --- litellm/integrations/slack_alerting.py | 48 ++++++++++++++++++++++++++ litellm/proxy/utils.py | 2 ++ 2 files changed, 50 insertions(+) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index d03922bc1f..f502b6675d 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -883,3 +883,51 @@ Model Info: ) # shuffle to prevent collisions await asyncio.sleep(interval) return + + async def send_weekly_spend_report(self): + """ """ + try: + await asyncio.sleep(10) + + from litellm.proxy.proxy_server import _get_weekly_spend_reports + + weekly_spend_per_team, weekly_spend_per_tag = ( + await _get_weekly_spend_reports() + ) + todays_date = datetime.datetime.now().date() + week_before = todays_date - datetime.timedelta(days=7) + + todays_date = todays_date.strftime("%m-%d-%Y") + week_before = week_before.strftime("%m-%d-%Y") + + _weekly_spend_message = ( + f"*💸 Weekly Spend Report for `{week_before} - {todays_date}` *\n" + ) + + if weekly_spend_per_team is not None: + _weekly_spend_message += "\n*Team Spend Report:*\n" + for spend in weekly_spend_per_team: + _team_spend = spend["total_spend"] + _team_spend = float(_team_spend) + # round to 4 decimal places + _team_spend = round(_team_spend, 4) + _weekly_spend_message += ( + f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n" + ) + + if weekly_spend_per_tag is not None: + _weekly_spend_message += "\n*Tag Spend Report:*\n" + for spend in weekly_spend_per_tag: + _tag_spend = spend["total_spend"] + _tag_spend = float(_tag_spend) + # round to 4 decimal places + _tag_spend = round(_tag_spend, 4) + _weekly_spend_message += f"Tag: `{spend['individual_request_tag']}` | Spend: `${_tag_spend}`\n" + + await self.send_alert( + message=_weekly_spend_message, + level="Low", + alert_type="daily_reports", + ) + except Exception as e: + verbose_proxy_logger.error("Error sending weekly spend report", e) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a97864f000..474b420b93 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -122,6 +122,8 @@ class ProxyLogging: alerting_args=alerting_args, ) + asyncio.create_task(self.slack_alerting_instance.send_weekly_spend_report()) + if "daily_reports" in self.alert_types: litellm.callbacks.append(self.slack_alerting_instance) # type: ignore From 63e41765021400cfaab0e263e2e377be8ff79592 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 09:26:51 -0700 Subject: [PATCH 02/11] feat - _get_weekly_spend_reports --- litellm/proxy/proxy_server.py | 53 ++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a9862022f8..99f44774b5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5415,6 +5415,57 @@ async def global_view_spend_tags( ) +async def _get_weekly_spend_reports(): + global prisma_client + todays_date = datetime.now().date() + _week_before = todays_date - timedelta(days=7) + + todays_date = todays_date.strftime("%Y-%m-%d") + _week_before = _week_before.strftime("%Y-%m-%d") + + try: + + sql_query = """ + SELECT + t.team_alias, + SUM(s.spend) AS total_spend, + s."startTime"::DATE AS log_date + FROM + "LiteLLM_SpendLogs" s + LEFT JOIN + "LiteLLM_TeamTable" t ON s.team_id = t.team_id + WHERE + s."startTime"::DATE BETWEEN $1::date AND $1::date + GROUP BY + t.team_alias, + log_date + ORDER BY + total_spend DESC; + """ + response = await prisma_client.db.query_raw( + sql_query, todays_date, _week_before + ) + + # get spend per tag for today + sql_query = """ + SELECT + jsonb_array_elements_text(request_tags) AS individual_request_tag, + "startTime"::DATE AS log_date, + SUM(spend) AS total_spend + FROM "LiteLLM_SpendLogs" + WHERE "startTime"::DATE BETWEEN $1::date AND $1::date + GROUP BY individual_request_tag, log_date; + """ + + spend_per_tag = await prisma_client.db.query_raw( + sql_query, todays_date, _week_before + ) + + return response, spend_per_tag + except Exception as e: + verbose_proxy_logger.error("Exception in _get_daily_spend_reports", e) # noqa + + @router.post( "/spend/calculate", tags=["Budget & Spend Tracking"], @@ -5802,7 +5853,7 @@ async def global_spend_keys( tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], ) -async def global_spend_per_tea(): +async def global_spend_per_team(): """ [BETA] This is a beta endpoint. It will change. From 09c064c94c13a892dfeee42a63f2d73c0bfcba8d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:10:44 -0700 Subject: [PATCH 03/11] feat - show monthly spend reports --- litellm/proxy/proxy_server.py | 36 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 99f44774b5..bddb9fc3b8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5415,50 +5415,48 @@ async def global_view_spend_tags( ) -async def _get_weekly_spend_reports(): +async def _get_spend_report_for_time_range( + start_date: str, + end_date: str, +): global prisma_client - todays_date = datetime.now().date() - _week_before = todays_date - timedelta(days=7) - - todays_date = todays_date.strftime("%Y-%m-%d") - _week_before = _week_before.strftime("%Y-%m-%d") + if prisma_client is None: + verbose_proxy_logger.error( + f"Database not connected. Connect a database to your proxy for weekly, monthly spend reports" + ) + return None try: - sql_query = """ SELECT t.team_alias, - SUM(s.spend) AS total_spend, - s."startTime"::DATE AS log_date + SUM(s.spend) AS total_spend FROM "LiteLLM_SpendLogs" s LEFT JOIN "LiteLLM_TeamTable" t ON s.team_id = t.team_id WHERE - s."startTime"::DATE BETWEEN $1::date AND $1::date + s."startTime"::DATE >= $1::date AND s."startTime"::DATE <= $2::date GROUP BY - t.team_alias, - log_date + t.team_alias ORDER BY total_spend DESC; """ - response = await prisma_client.db.query_raw( - sql_query, todays_date, _week_before - ) + response = await prisma_client.db.query_raw(sql_query, start_date, end_date) # get spend per tag for today sql_query = """ SELECT jsonb_array_elements_text(request_tags) AS individual_request_tag, - "startTime"::DATE AS log_date, SUM(spend) AS total_spend FROM "LiteLLM_SpendLogs" - WHERE "startTime"::DATE BETWEEN $1::date AND $1::date - GROUP BY individual_request_tag, log_date; + WHERE "startTime"::DATE >= $1::date AND "startTime"::DATE <= $2::date + GROUP BY individual_request_tag + ORDER BY total_spend DESC; """ spend_per_tag = await prisma_client.db.query_raw( - sql_query, todays_date, _week_before + sql_query, start_date, end_date ) return response, spend_per_tag From 2694721e814da50c04cd97ce1e097e81597ba047 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:17:09 -0700 Subject: [PATCH 04/11] fix - show monthly spend in slack reports --- litellm/integrations/slack_alerting.py | 70 ++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index f502b6675d..8df943098a 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -889,21 +889,20 @@ Model Info: try: await asyncio.sleep(10) - from litellm.proxy.proxy_server import _get_weekly_spend_reports + from litellm.proxy.proxy_server import _get_spend_report_for_time_range - weekly_spend_per_team, weekly_spend_per_tag = ( - await _get_weekly_spend_reports() - ) todays_date = datetime.datetime.now().date() week_before = todays_date - datetime.timedelta(days=7) - todays_date = todays_date.strftime("%m-%d-%Y") - week_before = week_before.strftime("%m-%d-%Y") - - _weekly_spend_message = ( - f"*💸 Weekly Spend Report for `{week_before} - {todays_date}` *\n" + weekly_spend_per_team, weekly_spend_per_tag = ( + await _get_spend_report_for_time_range( + start_date=week_before.strftime("%Y-%m-%d"), + end_date=todays_date.strftime("%Y-%m-%d"), + ) ) + _weekly_spend_message = f"*💸 Weekly Spend Report for `{week_before.strftime('%m-%d-%Y')} - {todays_date.strftime('%m-%d-%Y')}` *\n" + if weekly_spend_per_team is not None: _weekly_spend_message += "\n*Team Spend Report:*\n" for spend in weekly_spend_per_team: @@ -931,3 +930,56 @@ Model Info: ) except Exception as e: verbose_proxy_logger.error("Error sending weekly spend report", e) + + async def send_monthly_spend_report(self): + """ """ + try: + from calendar import monthrange + + await asyncio.sleep(10) + + from litellm.proxy.proxy_server import _get_spend_report_for_time_range + + todays_date = datetime.datetime.now().date() + first_day_of_month = todays_date.replace(day=1) + _, last_day_of_month = monthrange(todays_date.year, todays_date.month) + last_day_of_month = first_day_of_month + datetime.timedelta( + days=last_day_of_month - 1 + ) + + monthly_spend_per_team, monthly_spend_per_tag = ( + await _get_spend_report_for_time_range( + start_date=first_day_of_month.strftime("%Y-%m-%d"), + end_date=last_day_of_month.strftime("%Y-%m-%d"), + ) + ) + + _spend_message = f"*💸 Monthly Spend Report for `{first_day_of_month.strftime('%m-%d-%Y')} - {last_day_of_month.strftime('%m-%d-%Y')}` *\n" + + if monthly_spend_per_team is not None: + _spend_message += "\n*Team Spend Report:*\n" + for spend in monthly_spend_per_team: + _team_spend = spend["total_spend"] + _team_spend = float(_team_spend) + # round to 4 decimal places + _team_spend = round(_team_spend, 4) + _spend_message += ( + f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n" + ) + + if monthly_spend_per_tag is not None: + _spend_message += "\n*Tag Spend Report:*\n" + for spend in monthly_spend_per_tag: + _tag_spend = spend["total_spend"] + _tag_spend = float(_tag_spend) + # round to 4 decimal places + _tag_spend = round(_tag_spend, 4) + _spend_message += f"Tag: `{spend['individual_request_tag']}` | Spend: `${_tag_spend}`\n" + + await self.send_alert( + message=_spend_message, + level="Low", + alert_type="daily_reports", + ) + except Exception as e: + verbose_proxy_logger.error("Error sending weekly spend report", e) From a86f9d7b9387e0f3eae9ec5c5a699d0a89dfc6a9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:17:40 -0700 Subject: [PATCH 05/11] feat - send_monthly_spend_report --- litellm/proxy/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 474b420b93..8338b66125 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -124,6 +124,8 @@ class ProxyLogging: asyncio.create_task(self.slack_alerting_instance.send_weekly_spend_report()) + asyncio.create_task(self.slack_alerting_instance.send_monthly_spend_report()) + if "daily_reports" in self.alert_types: litellm.callbacks.append(self.slack_alerting_instance) # type: ignore From fa94632bc261a01444976c6b00d47fae93b48d02 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:44:19 -0700 Subject: [PATCH 06/11] schedule weekly/monthly spend reports --- litellm/proxy/proxy_server.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bddb9fc3b8..624e2b0c0d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3479,6 +3479,25 @@ async def startup_event(): await proxy_config.add_deployment( prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj ) + + if ( + proxy_logging_obj is not None + and proxy_logging_obj.slack_alerting_instance is not None + ): + print("Alerting: Initializing Weekly/Monthly Spend Reports") # noqa + ### Schedule weekly/monhtly spend reports ### + scheduler.add_job( + proxy_logging_obj.slack_alerting_instance.send_weekly_spend_report, + "cron", + day_of_week="mon", + ) + + scheduler.add_job( + proxy_logging_obj.slack_alerting_instance.send_monthly_spend_report, + "cron", + day=1, + ) + scheduler.start() From f3088988fc3cf09392eecf035ea5c11eddeee1f2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:45:22 -0700 Subject: [PATCH 07/11] fix scheduling spend reports --- litellm/proxy/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8338b66125..a97864f000 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -122,10 +122,6 @@ class ProxyLogging: alerting_args=alerting_args, ) - asyncio.create_task(self.slack_alerting_instance.send_weekly_spend_report()) - - asyncio.create_task(self.slack_alerting_instance.send_monthly_spend_report()) - if "daily_reports" in self.alert_types: litellm.callbacks.append(self.slack_alerting_instance) # type: ignore From 77191ff98adc3e05203f28cf527a98e1a1846d52 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:50:26 -0700 Subject: [PATCH 08/11] test - weekly / monthly spend report alerts on /health/services --- litellm/proxy/proxy_server.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 624e2b0c0d..5d6a2acb08 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9555,6 +9555,14 @@ async def health_services_endpoint( level="Low", alert_type="budget_alerts", ) + + if prisma_client is not None: + asyncio.create_task( + proxy_logging_obj.slack_alerting_instance.send_monthly_spend_report() + ) + asyncio.create_task( + proxy_logging_obj.slack_alerting_instance.send_weekly_spend_report() + ) return { "status": "success", "message": "Mock Slack Alert sent, verify Slack Alert Received on your channel", From 4d72d4676de9054eb55006836643b6a4f5d289c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:51:59 -0700 Subject: [PATCH 09/11] fix - spend reports on alerts --- litellm/integrations/slack_alerting.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 8df943098a..012033b018 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -887,8 +887,6 @@ Model Info: async def send_weekly_spend_report(self): """ """ try: - await asyncio.sleep(10) - from litellm.proxy.proxy_server import _get_spend_report_for_time_range todays_date = datetime.datetime.now().date() @@ -936,8 +934,6 @@ Model Info: try: from calendar import monthrange - await asyncio.sleep(10) - from litellm.proxy.proxy_server import _get_spend_report_for_time_range todays_date = datetime.datetime.now().date() From 968c7256c9999d687a396e019567fff4b1107317 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 10:55:54 -0700 Subject: [PATCH 10/11] ui - show daily reports on UI --- ui/litellm-dashboard/src/components/settings.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index ff7e07005d..96c20bf15c 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -109,6 +109,7 @@ const Settings: React.FC = ({ "llm_requests_hanging": "LLM Requests Hanging", "budget_alerts": "Budget Alerts (API Keys, Users)", "db_exceptions": "Database Exceptions (Read/Write)", + "daily_reports": "Weekly/Monthly Spend Reports", } useEffect(() => { From 9da8443fd71a4091cb97cb0c7ad13e715b092382 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 13 May 2024 12:30:54 -0700 Subject: [PATCH 11/11] fix - only schedule spend alerting when db is not none --- litellm/proxy/proxy_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5d6a2acb08..837940828c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3483,6 +3483,7 @@ async def startup_event(): if ( proxy_logging_obj is not None and proxy_logging_obj.slack_alerting_instance is not None + and prisma_client is not None ): print("Alerting: Initializing Weekly/Monthly Spend Reports") # noqa ### Schedule weekly/monhtly spend reports ###