Merge branch 'main' into litellm_add_model_api_fix

.github/workflows/ghcr_deploy.yml

@@ -18,6 +18,23 @@ env:
 
 # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
+  determine-release-type:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set release type
+        id: set-release-type
+        run: |
+          if [ "${{ github.event.inputs.stable }}" == "true" ]; then
+            echo "stable release"
+            echo "type_release=stable" >> $GITHUB_ENV
+          else
+            echo "latest release"
+            echo "type_release=latest" >> $GITHUB_ENV
+          fi
+          echo "type_release value: $type_release"
+        env:
+          ACTIONS_ALLOW_ENV_MARK: true
+
   docker-hub-deploy:
     if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
@@ -91,7 +108,7 @@ jobs:
         with:
           context: .
           push: true
-          tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.stable && 'stable' || 'latest' }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
+          tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ env.type_release }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
           labels: ${{ steps.meta.outputs.labels }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
           
@@ -128,7 +145,7 @@ jobs:
           context: .
           file: Dockerfile.database
           push: true
-          tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.stable && 'stable' || 'latest' }}
+          tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ env.type_release }} 
           labels: ${{ steps.meta-database.outputs.labels }} 
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
   
@@ -165,8 +182,7 @@ jobs:
           context: .
           file: ./litellm-js/spend-logs/Dockerfile
           push: true
-          tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.stable && 'stable' || 'latest' }}
+          tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ env.type_release }}
-          labels: ${{ steps.meta-spend-logs.outputs.labels }} 
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
   build-and-push-helm-chart:

enterprise/utils.py

@@ -1,5 +1,6 @@
 # Enterprise Proxy Util Endpoints
 from litellm._logging import verbose_logger
+import collections
 
 
 async def get_spend_by_tags(start_date=None, end_date=None, prisma_client=None):
@@ -17,6 +18,48 @@ async def get_spend_by_tags(start_date=None, end_date=None, prisma_client=None):
     return response
 
 
+async def ui_get_spend_by_tags(start_date=None, end_date=None, prisma_client=None):
+    response = await prisma_client.db.query_raw(
+        """
+        SELECT
+        jsonb_array_elements_text(request_tags) AS individual_request_tag,
+        DATE(s."startTime") AS spend_date,
+        COUNT(*) AS log_count,
+        SUM(spend) AS total_spend
+        FROM "LiteLLM_SpendLogs" s
+        WHERE s."startTime" >= current_date - interval '30 days'
+        GROUP BY individual_request_tag, spend_date
+        ORDER BY spend_date;
+        """
+    )
+
+    # print("tags - spend")
+    # print(response)
+    # Bar Chart 1 - Spend per tag - Top 10 tags by spend
+    total_spend_per_tag = collections.defaultdict(float)
+    total_requests_per_tag = collections.defaultdict(int)
+    for row in response:
+        tag_name = row["individual_request_tag"]
+        tag_spend = row["total_spend"]
+
+        total_spend_per_tag[tag_name] += tag_spend
+        total_requests_per_tag[tag_name] += row["log_count"]
+
+    sorted_tags = sorted(total_spend_per_tag.items(), key=lambda x: x[1], reverse=True)
+    # convert to ui format
+    ui_tags = []
+    for tag in sorted_tags:
+        ui_tags.append(
+            {
+                "name": tag[0],
+                "value": tag[1],
+                "log_count": total_requests_per_tag[tag[0]],
+            }
+        )
+
+    return {"top_10_tags": ui_tags}
+
+
 async def view_spend_logs_from_clickhouse(
     api_key=None, user_id=None, request_id=None, start_date=None, end_date=None
 ):

litellm/proxy/_experimental/out/_next/static/oEHk25X5wW-wbmybW4h4d/_buildManifest.js

@@ -0,0 +1 @@
+self.__BUILD_MANIFEST={__rewrites:{afterFiles:[],beforeFiles:[],fallback:[]},"/_error":["static/chunks/pages/_error-d6107f1aac0c574c.js"],sortedPages:["/_app","/_error"]},self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();

litellm/proxy/_experimental/out/_next/static/oEHk25X5wW-wbmybW4h4d/_ssgManifest.js

@@ -0,0 +1 @@
+self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()

litellm/proxy/proxy_server.py

@@ -2416,9 +2416,10 @@ class ProxyConfig:
                             )
                     _litellm_params = LiteLLM_Params(**_litellm_params)
                 else:
-                    raise Exception(
+                    verbose_proxy_logger.error(
                         f"Invalid model added to proxy db. Invalid litellm params. litellm_params={_litellm_params}"
                     )
+                    continue  # skip to next model
 
                 if m.model_info is not None and isinstance(m.model_info, dict):
                     if "id" not in m.model_info:
@@ -2437,7 +2438,7 @@ class ProxyConfig:
 
             llm_model_list = llm_router.get_model_list()
         except Exception as e:
-            raise e
+            verbose_proxy_logger.error("{}".format(str(e)))
 
 
 proxy_config = ProxyConfig()
@@ -4754,6 +4755,73 @@ async def view_spend_tags(
         )
 
 
+@router.get(
+    "/global/spend/tags",
+    tags=["Budget & Spend Tracking"],
+    dependencies=[Depends(user_api_key_auth)],
+    include_in_schema=False,
+    responses={
+        200: {"model": List[LiteLLM_SpendLogs]},
+    },
+)
+async def global_view_spend_tags(
+    start_date: Optional[str] = fastapi.Query(
+        default=None,
+        description="Time from which to start viewing key spend",
+    ),
+    end_date: Optional[str] = fastapi.Query(
+        default=None,
+        description="Time till which to view key spend",
+    ),
+):
+    """
+    LiteLLM Enterprise - View Spend Per Request Tag. Used by LiteLLM UI
+
+    Example Request:
+    ```
+    curl -X GET "http://0.0.0.0:8000/spend/tags" \
+-H "Authorization: Bearer sk-1234"
+    ```
+
+    Spend with Start Date and End Date
+    ```
+    curl -X GET "http://0.0.0.0:8000/spend/tags?start_date=2022-01-01&end_date=2022-02-01" \
+-H "Authorization: Bearer sk-1234"
+    ```
+    """
+
+    from enterprise.utils import ui_get_spend_by_tags
+
+    global prisma_client
+    try:
+        if prisma_client is None:
+            raise Exception(
+                f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
+            )
+
+        response = await ui_get_spend_by_tags(
+            start_date=start_date, end_date=end_date, prisma_client=prisma_client
+        )
+
+        return response
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "detail", f"/spend/tags Error({str(e)})"),
+                type="internal_error",
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
+            )
+        elif isinstance(e, ProxyException):
+            raise e
+        raise ProxyException(
+            message="/spend/tags Error" + str(e),
+            type="internal_error",
+            param=getattr(e, "param", "None"),
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+
+
 @router.post(
     "/spend/calculate",
     tags=["Budget & Spend Tracking"],

litellm/tests/test_amazing_s3_logs.py

@@ -15,6 +15,7 @@ import time, random
 import pytest
 
 
+@pytest.mark.skip(reason="AWS Suspended Account")
 def test_s3_logging():
     # all s3 requests need to be in one test function
     # since we are modifying stdout, and pytests runs tests in parallel
@@ -124,6 +125,7 @@ def test_s3_logging():
 # test_s3_logging()
 
 
+@pytest.mark.skip(reason="AWS Suspended Account")
 def test_s3_logging_async():
     # this tests time added to make s3 logging calls, vs just acompletion calls
     try:

tests/test_models.py

@@ -137,6 +137,7 @@ async def test_add_models():
         key_gen = await generate_key(session=session)
         key = key_gen["key"]
         await add_models(session=session)
+        await asyncio.sleep(60)
         await chat_completion(session=session, key=key)
 
 

ui/litellm-dashboard/out/_next/static/oEHk25X5wW-wbmybW4h4d/_buildManifest.js

@@ -0,0 +1 @@
+self.__BUILD_MANIFEST={__rewrites:{afterFiles:[],beforeFiles:[],fallback:[]},"/_error":["static/chunks/pages/_error-d6107f1aac0c574c.js"],sortedPages:["/_app","/_error"]},self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();

ui/litellm-dashboard/out/_next/static/oEHk25X5wW-wbmybW4h4d/_ssgManifest.js

@@ -0,0 +1 @@
+self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()

ui/litellm-dashboard/src/components/networking.tsx

@@ -473,6 +473,36 @@ export const teamSpendLogsCall = async (accessToken: String) => {
   }
 };
 
+
+export const tagsSpendLogsCall = async (accessToken: String) => {
+  try {
+    const url = proxyBaseUrl
+      ? `${proxyBaseUrl}/global/spend/tags`
+      : `/global/spend/tags`;
+    console.log("in tagsSpendLogsCall:", url);
+    const response = await fetch(`${url}`, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+    if (!response.ok) {
+      const errorData = await response.text();
+      message.error(errorData);
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    console.log(data);
+    return data;
+  } catch (error) {
+    console.error("Failed to create key:", error);
+    throw error;
+  }
+};
+
+
 export const userSpendLogsCall = async (
   accessToken: String,
   token: String,

ui/litellm-dashboard/src/components/usage.tsx

@@ -1,4 +1,4 @@
-import { BarChart, BarList, Card, Title } from "@tremor/react";
+import { BarChart, BarList, Card, Title, Table, TableHead, TableHeaderCell, TableRow, TableCell, TableBody, Metric } from "@tremor/react";
 
 import React, { useState, useEffect } from "react";
 
@@ -11,6 +11,7 @@ import {
   adminTopKeysCall,
   adminTopModelsCall,
   teamSpendLogsCall,
+  tagsSpendLogsCall
 } from "./networking";
 import { start } from "repl";
 
@@ -139,6 +140,7 @@ const UsagePage: React.FC<UsagePageProps> = ({
   const [topModels, setTopModels] = useState<any[]>([]);
   const [topUsers, setTopUsers] = useState<any[]>([]);
   const [teamSpendData, setTeamSpendData] = useState<any[]>([]);
+  const [topTagsData, setTopTagsData] = useState<any[]>([]);
   const [uniqueTeamIds, setUniqueTeamIds] = useState<any[]>([]);
   const [totalSpendPerTeam, setTotalSpendPerTeam] = useState<any[]>([]);
 
@@ -217,6 +219,10 @@ const UsagePage: React.FC<UsagePageProps> = ({
             })
 
             setTotalSpendPerTeam(total_spend_per_team);
+
+            //get top tags
+            const top_tags = await tagsSpendLogsCall(accessToken);
+            setTopTagsData(top_tags.top_10_tags);
           } else if (userRole == "App Owner") {
             await userSpendLogsCall(
               accessToken,
@@ -273,6 +279,7 @@ const UsagePage: React.FC<UsagePageProps> = ({
         <TabList className="mt-2">
           <Tab>All Up</Tab>
           <Tab>Team Based Usage</Tab>
+           <Tab>Tag Based Usage</Tab>
         </TabList>
         <TabPanels>
           <TabPanel>
@@ -371,6 +378,47 @@ const UsagePage: React.FC<UsagePageProps> = ({
               </Col>
             </Grid>
             </TabPanel>
+            <TabPanel>
+            <Grid numItems={2} className="gap-2 h-[75vh] w-full mb-4">
+              <Col numColSpan={2}>
+
+              <Card>
+              <Title>Spend Per Tag - Last 30 Days</Title>
+              <Text>Get Started Tracking cost per tag <a href="https://docs.litellm.ai/docs/proxy/enterprise#tracking-spend-for-custom-tags" target="_blank">here</a></Text>
+              <Table>
+                <TableHead>
+                  <TableRow>
+                    <TableHeaderCell>Tag</TableHeaderCell>
+                    <TableHeaderCell>Spend</TableHeaderCell>
+                    <TableHeaderCell>Requests</TableHeaderCell>
+                  </TableRow>
+                </TableHead>
+                <TableBody>
+                  {topTagsData.map((tag) => (
+                    <TableRow key={tag.name}>
+                      <TableCell>{tag.name}</TableCell>
+                      <TableCell>{tag.value}</TableCell>
+                      <TableCell>{tag.log_count}</TableCell>
+                    </TableRow>
+                  ))}
+                </TableBody>
+              </Table>
+                {/* <BarChart
+                  className="h-72"
+                  data={teamSpendData}
+                  showLegend={true}
+                  index="date"
+                  categories={uniqueTeamIds}
+                  yAxisWidth={80}
+                  
+                  stack={true}
+                /> */}
+              </Card>
+              </Col>
+              <Col numColSpan={2}>
+              </Col>
+            </Grid>
+            </TabPanel>
         </TabPanels>
       </TabGroup>
     </div>