From 4821fa9201af5ee3da1cf337e3ba34c642ae6eee Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 10:04:34 -0800
Subject: [PATCH 01/23] (v0) add schema.prisma

---
 litellm/proxy/_types.py     | 16 ++++++++++++++++
 litellm/proxy/schema.prisma | 15 +++++++++++++++
 schema.prisma               | 15 +++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 6e85883141..220bf2f919 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -194,6 +194,7 @@ class DynamoDBArgs(LiteLLMBase):
     user_table_name: str = "LiteLLM_UserTable"
     key_table_name: str = "LiteLLM_VerificationToken"
     config_table_name: str = "LiteLLM_Config"
+    spend_table_name: str = "LiteLLM_SpendLogs"
 
 
 class ConfigGeneralSettings(LiteLLMBase):
@@ -312,3 +313,18 @@ class LiteLLM_UserTable(LiteLLMBase):
         if values.get("models") is None:
             values.update({"models", []})
         return values
+
+
+class LiteLLM_SpendLogs(LiteLLMBase):
+    id: str
+    call_type: str
+    startTime: Union[str, None]
+    endTime: Union[str, None]
+    model: str = ""
+    user: str = ""
+    modelParameters: Dict = {}
+    messages: List[str] = []
+    call_cost: float = 0.0
+    response: Dict = {}
+    usage: Dict = {}
+    metadata: Dict = {}
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index aa45a88186..d2e338bd4c 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -31,4 +31,19 @@ model LiteLLM_VerificationToken {
 model LiteLLM_Config {
   param_name String @id
   param_value Json?
+}
+
+model LiteLLM_SpendLogs {
+  request_id          String @unique
+  call_type           String
+  startTime           DateTime // Assuming start_time is a DateTime field
+  endTime             DateTime // Assuming end_time is a DateTime field
+  model               String   @default("")
+  user                String   @default("")
+  modelParameters     Json     @default("{}")// Assuming optional_params is a JSON field
+  messages            Json     @default("[]")
+  spend               Float    @default(0.0)
+  response            Json     @default("{}")
+  usage               Json     @default("{}")
+  metadata            Json     @default("{}")
 }
\ No newline at end of file
diff --git a/schema.prisma b/schema.prisma
index 704ada42c9..df2c1d0b4b 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -31,4 +31,19 @@ model LiteLLM_VerificationToken {
 model LiteLLM_Config {
   param_name String @id
   param_value Json?
+}
+
+model LiteLLM_SpendLogs {
+  id                  String @unique
+  call_type           String
+  startTime           DateTime // Assuming start_time is a DateTime field
+  endTime             DateTime // Assuming end_time is a DateTime field
+  model               String   @default("")
+  user                String   @default("")
+  modelParameters     Json     @default("{}")// Assuming optional_params is a JSON field
+  messages            Json     @default("[]")
+  call_cost           Float    @default(0.0)
+  response            Json     @default("{}")
+  usage               Json     @default("{}")
+  metadata            Json     @default("{}")
 }
\ No newline at end of file

From 4a5f987512e94e46b542c1929c799dea99380aeb Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 10:09:02 -0800
Subject: [PATCH 02/23] (feat) insert_data to spend table

---
 litellm/proxy/_types.py |  2 +-
 litellm/proxy/utils.py  | 27 +++++++++++++++++++++++----
 schema.prisma           |  4 ++--
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 220bf2f919..cf8350022b 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -316,7 +316,7 @@ class LiteLLM_UserTable(LiteLLMBase):
 
 
 class LiteLLM_SpendLogs(LiteLLMBase):
-    id: str
+    request_id: str
     call_type: str
     startTime: Union[str, None]
     endTime: Union[str, None]
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index ab1fea463e..f24412a84c 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -1,7 +1,12 @@
 from typing import Optional, List, Any, Literal, Union
 import os, subprocess, hashlib, importlib, asyncio, copy, json, aiohttp, httpx
 import litellm, backoff
-from litellm.proxy._types import UserAPIKeyAuth, DynamoDBArgs, LiteLLM_VerificationToken
+from litellm.proxy._types import (
+    UserAPIKeyAuth,
+    DynamoDBArgs,
+    LiteLLM_VerificationToken,
+    LiteLLM_SpendLogs,
+)
 from litellm.caching import DualCache
 from litellm.proxy.hooks.parallel_request_limiter import MaxParallelRequestsHandler
 from litellm.proxy.hooks.max_budget_limiter import MaxBudgetLimiter
@@ -316,7 +321,7 @@ class PrismaClient:
         self,
         key: str,
         value: Any,
-        table_name: Literal["users", "keys", "config"],
+        table_name: Literal["users", "keys", "config", "spend"],
     ):
         """
         Generic implementation of get data
@@ -334,6 +339,10 @@ class PrismaClient:
                 response = await self.db.litellm_config.find_first(  # type: ignore
                     where={key: value}  # type: ignore
                 )
+            elif table_name == "spend":
+                response = await self.db.l.find_first(  # type: ignore
+                    where={key: value}  # type: ignore
+                )
             return response
         except Exception as e:
             asyncio.create_task(
@@ -417,7 +426,7 @@ class PrismaClient:
         on_backoff=on_backoff,  # specifying the function to call on backoff
     )
     async def insert_data(
-        self, data: dict, table_name: Literal["user", "key", "config"]
+        self, data: dict, table_name: Literal["user", "key", "config", "spend"]
     ):
         """
         Add a key to the database. If it already exists, do nothing.
@@ -473,8 +482,18 @@ class PrismaClient:
                     )
 
                     tasks.append(updated_table_row)
-
                 await asyncio.gather(*tasks)
+            elif table_name == "spend":
+                db_data = self.jsonify_object(data=data)
+                new_spend_row = await self.db.litellm_spendlogs.upsert(
+                    where={"request_id": data["request_id"]},
+                    data={
+                        "create": {**db_data},  # type: ignore
+                        "update": {},  # don't do anything if it already exists
+                    },
+                )
+                return new_spend_row
+
         except Exception as e:
             print_verbose(f"LiteLLM Prisma Client Exception: {e}")
             asyncio.create_task(
diff --git a/schema.prisma b/schema.prisma
index df2c1d0b4b..ed69f67a76 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -34,7 +34,7 @@ model LiteLLM_Config {
 }
 
 model LiteLLM_SpendLogs {
-  id                  String @unique
+  request_id          String @unique
   call_type           String
   startTime           DateTime // Assuming start_time is a DateTime field
   endTime             DateTime // Assuming end_time is a DateTime field
@@ -42,7 +42,7 @@ model LiteLLM_SpendLogs {
   user                String   @default("")
   modelParameters     Json     @default("{}")// Assuming optional_params is a JSON field
   messages            Json     @default("[]")
-  call_cost           Float    @default(0.0)
+  spend               Float    @default(0.0)
   response            Json     @default("{}")
   usage               Json     @default("{}")
   metadata            Json     @default("{}")

From d14d36af9add19434f96a6ef49482c18acc610b3 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 11:54:15 -0800
Subject: [PATCH 03/23] (v0 ) working - writing /chat/completion spend tracking

---
 litellm/proxy/_types.py         | 27 +++++++-------
 litellm/proxy/proxy_config.yaml |  4 +--
 litellm/proxy/proxy_server.py   | 35 ++++++++++++++++--
 litellm/proxy/schema.prisma     |  1 +
 litellm/utils.py                | 63 +++++++++++++++++++++++++++++++++
 schema.prisma                   |  1 +
 6 files changed, 114 insertions(+), 17 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index cf8350022b..9bc6b09b12 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1,8 +1,8 @@
-from pydantic import BaseModel, Extra, Field, root_validator
+from pydantic import BaseModel, Extra, Field, root_validator, Json
 import enum
-from typing import Optional, List, Union, Dict, Literal
+from typing import Optional, List, Union, Dict, Literal, Any
 from datetime import datetime
-import uuid, json
+import uuid, json, sys, os
 
 
 class LiteLLMBase(BaseModel):
@@ -318,13 +318,14 @@ class LiteLLM_UserTable(LiteLLMBase):
 class LiteLLM_SpendLogs(LiteLLMBase):
     request_id: str
     call_type: str
-    startTime: Union[str, None]
-    endTime: Union[str, None]
-    model: str = ""
-    user: str = ""
-    modelParameters: Dict = {}
-    messages: List[str] = []
-    call_cost: float = 0.0
-    response: Dict = {}
-    usage: Dict = {}
-    metadata: Dict = {}
+    startTime: Union[str, datetime, None]
+    endTime: Union[str, datetime, None]
+    model: Optional[str] = ""
+    user: Optional[str] = ""
+    modelParameters: Optional[Json] = {}
+    messages: Optional[Json] = []
+    spend: Optional[float] = 0.0
+    response: Optional[Json] = {}
+    usage: Optional[Json] = {}
+    metadata: Optional[Json] = {}
+    cache_hit: Optional[str] = "False"
diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index 5b87ab775b..8cd2fcec85 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -61,8 +61,8 @@ litellm_settings:
   # setting callback class
   # callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance]
 
-# general_settings: 
-  # master_key: sk-1234
+general_settings: 
+  master_key: sk-1234
   # database_type: "dynamo_db" 
   # database_args: { # 👈  all args - https://github.com/BerriAI/litellm/blob/befbcbb7ac8f59835ce47415c128decf37aac328/litellm/proxy/_types.py#L190
   #   "billing_mode": "PAY_PER_REQUEST", 
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index d3667892bb..fdc81f88e0 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -510,6 +510,7 @@ async def track_cost_callback(
     global prisma_client, custom_db_client
     try:
         # check if it has collected an entire stream response
+        verbose_proxy_logger.debug(f"Proxy: In track_cost_callback for {kwargs}")
         verbose_proxy_logger.debug(
             f"kwargs stream: {kwargs.get('stream', None)} + complete streaming response: {kwargs.get('complete_streaming_response', None)}"
         )
@@ -546,13 +547,27 @@ async def track_cost_callback(
                 prisma_client is not None or custom_db_client is not None
             ):
                 await update_database(
-                    token=user_api_key, response_cost=response_cost, user_id=user_id
+                    token=user_api_key,
+                    response_cost=response_cost,
+                    user_id=user_id,
+                    kwargs=kwargs,
+                    completion_response=completion_response,
+                    start_time=start_time,
+                    end_time=end_time,
                 )
     except Exception as e:
         verbose_proxy_logger.debug(f"error in tracking cost callback - {str(e)}")
 
 
-async def update_database(token, response_cost, user_id=None):
+async def update_database(
+    token,
+    response_cost,
+    user_id=None,
+    kwargs=None,
+    completion_response=None,
+    start_time=None,
+    end_time=None,
+):
     try:
         verbose_proxy_logger.debug(
             f"Enters prisma db call, token: {token}; user_id: {user_id}"
@@ -622,9 +637,25 @@ async def update_database(token, response_cost, user_id=None):
                     key=token, value={"spend": new_spend}, table_name="key"
                 )
 
+        async def _insert_spend_log_to_db():
+            # Helper to generate payload to log
+            verbose_proxy_logger.debug("inserting spend log to db")
+            payload = litellm.utils.get_logging_payload(
+                kwargs=kwargs,
+                response_obj=completion_response,
+                start_time=start_time,
+                end_time=end_time,
+            )
+
+            payload["spend"] = response_cost
+
+            if prisma_client is not None:
+                await prisma_client.insert_data(data=payload, table_name="spend")
+
         tasks = []
         tasks.append(_update_user_db())
         tasks.append(_update_key_db())
+        tasks.append(_insert_spend_log_to_db())
         await asyncio.gather(*tasks)
     except Exception as e:
         verbose_proxy_logger.debug(
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index d2e338bd4c..9049f953d7 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -46,4 +46,5 @@ model LiteLLM_SpendLogs {
   response            Json     @default("{}")
   usage               Json     @default("{}")
   metadata            Json     @default("{}")
+  cache_hit           String   @default("")
 }
\ No newline at end of file
diff --git a/litellm/utils.py b/litellm/utils.py
index f7cc5d2a54..b22a053ff5 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8423,3 +8423,66 @@ def print_args_passed_to_litellm(original_function, args, kwargs):
     except:
         # This should always be non blocking
         pass
+
+
+def get_logging_payload(kwargs, response_obj, start_time, end_time):
+    from litellm.proxy._types import LiteLLM_SpendLogs
+    from pydantic import Json
+
+    # standardize this function to be used across, s3, dynamoDB, langfuse logging
+    litellm_params = kwargs.get("litellm_params", {})
+    metadata = (
+        litellm_params.get("metadata", {}) or {}
+    )  # if litellm_params['metadata'] == None
+    messages = kwargs.get("messages")
+    optional_params = kwargs.get("optional_params", {})
+    call_type = kwargs.get("call_type", "litellm.completion")
+    cache_hit = kwargs.get("cache_hit", False)
+    usage = response_obj["usage"]
+    id = response_obj.get("id", str(uuid.uuid4()))
+
+    payload = {
+        "request_id": id,
+        "call_type": call_type,
+        "cache_hit": cache_hit,
+        "startTime": start_time,
+        "endTime": end_time,
+        "model": kwargs.get("model", ""),
+        "user": kwargs.get("user", ""),
+        "modelParameters": optional_params,
+        "messages": messages,
+        "response": response_obj,
+        "usage": usage,
+        "metadata": metadata,
+    }
+
+    json_fields = [
+        field
+        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
+        if field_type == Json or field_type == Optional[Json]
+    ]
+    str_fields = [
+        field
+        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
+        if field_type == str or field_type == Optional[str]
+    ]
+    datetime_fields = [
+        field
+        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
+        if field_type == datetime
+    ]
+
+    for param in json_fields:
+        if param in payload and type(payload[param]) != Json:
+            if type(payload[param]) == ModelResponse:
+                payload[param] = payload[param].model_dump_json()
+            elif type(payload[param]) == Usage:
+                payload[param] = payload[param].model_dump_json()
+            else:
+                payload[param] = json.dumps(payload[param])
+
+    for param in str_fields:
+        if param in payload and type(payload[param]) != str:
+            payload[param] = str(payload[param])
+
+    return payload
diff --git a/schema.prisma b/schema.prisma
index ed69f67a76..a07dcad08e 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -46,4 +46,5 @@ model LiteLLM_SpendLogs {
   response            Json     @default("{}")
   usage               Json     @default("{}")
   metadata            Json     @default("{}")
+  cache_hit           String   @default("")
 }
\ No newline at end of file

From b0e18e92b6cc5eaf9a511b71246459a508e8bb09 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 12:05:08 -0800
Subject: [PATCH 04/23] (fix) when kwargs==None

---
 litellm/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/litellm/utils.py b/litellm/utils.py
index b22a053ff5..1ff9dbe1c8 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8429,6 +8429,8 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time):
     from litellm.proxy._types import LiteLLM_SpendLogs
     from pydantic import Json
 
+    if kwargs == None:
+        kwargs = {}
     # standardize this function to be used across, s3, dynamoDB, langfuse logging
     litellm_params = kwargs.get("litellm_params", {})
     metadata = (

From 88cdfedf8429d3b705bd48301e4b0e783884854b Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 12:21:56 -0800
Subject: [PATCH 05/23] (feat) track cost streaming

---
 litellm/proxy/proxy_server.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index fdc81f88e0..10efd495b3 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -531,7 +531,13 @@ async def track_cost_callback(
                 prisma_client is not None or custom_db_client is not None
             ):
                 await update_database(
-                    token=user_api_key, response_cost=response_cost, user_id=user_id
+                    token=user_api_key,
+                    response_cost=response_cost,
+                    user_id=user_id,
+                    kwargs=kwargs,
+                    completion_response=completion_response,
+                    start_time=start_time,
+                    end_time=end_time,
                 )
         elif kwargs["stream"] == False:  # for non streaming responses
             response_cost = litellm.completion_cost(

From 4eb9221121d357dafafeed0381fc58747d51ed46 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 12:29:21 -0800
Subject: [PATCH 06/23] (fix) track EmbeddingResponse cost

---
 litellm/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/litellm/utils.py b/litellm/utils.py
index 1ff9dbe1c8..77f0c331e9 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8478,6 +8478,8 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time):
         if param in payload and type(payload[param]) != Json:
             if type(payload[param]) == ModelResponse:
                 payload[param] = payload[param].model_dump_json()
+            if type(payload[param]) == EmbeddingResponse:
+                payload[param] = payload[param].model_dump_json()
             elif type(payload[param]) == Usage:
                 payload[param] = payload[param].model_dump_json()
             else:

From 5b54bcc712c2d92f9709dc6961d8029ceedd3498 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 12:39:11 -0800
Subject: [PATCH 07/23] (feat) spendLogs table DynamoDB

---
 litellm/proxy/db/dynamo_db.py | 21 ++++++++++++++++++++-
 litellm/proxy/proxy_server.py |  3 +++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/db/dynamo_db.py b/litellm/proxy/db/dynamo_db.py
index eb1c085286..83cf6b1572 100644
--- a/litellm/proxy/db/dynamo_db.py
+++ b/litellm/proxy/db/dynamo_db.py
@@ -131,10 +131,27 @@ class DynamoDBWrapper(CustomDB):
                 raise Exception(
                     f"Failed to create table - {self.database_arguments.config_table_name}.\nPlease create a new table called {self.database_arguments.config_table_name}\nAND set `hash_key` as 'param_name'"
                 )
+
+            ## Spend
+            try:
+                verbose_proxy_logger.debug("DynamoDB Wrapper - Creating Spend Table")
+                error_occurred = False
+                table = client.table(self.database_arguments.spend_table_name)
+                if not await table.exists():
+                    await table.create(
+                        self.throughput_type,
+                        KeySchema(hash_key=KeySpec("request_id", KeyType.string)),
+                    )
+            except Exception as e:
+                error_occurred = True
+            if error_occurred == True:
+                raise Exception(
+                    f"Failed to create table - {self.database_arguments.key_table_name}.\nPlease create a new table called {self.database_arguments.key_table_name}\nAND set `hash_key` as 'token'"
+                )
             verbose_proxy_logger.debug("DynamoDB Wrapper - Done connecting()")
 
     async def insert_data(
-        self, value: Any, table_name: Literal["user", "key", "config"]
+        self, value: Any, table_name: Literal["user", "key", "config", "spend"]
     ):
         from aiodynamo.client import Client
         from aiodynamo.credentials import Credentials, StaticCredentials
@@ -166,6 +183,8 @@ class DynamoDBWrapper(CustomDB):
                 table = client.table(self.database_arguments.key_table_name)
             elif table_name == "config":
                 table = client.table(self.database_arguments.config_table_name)
+            elif table_name == "spend":
+                table = client.table(self.database_arguments.spend_table_name)
 
             for k, v in value.items():
                 if isinstance(v, datetime):
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 10efd495b3..32e985113e 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -658,6 +658,9 @@ async def update_database(
             if prisma_client is not None:
                 await prisma_client.insert_data(data=payload, table_name="spend")
 
+            elif custom_db_client is not None:
+                await custom_db_client.insert_data(payload, table_name="spend")
+
         tasks = []
         tasks.append(_update_user_db())
         tasks.append(_update_key_db())

From 73938080f26eaa1f8b71c1c596cb3aace2eb920d Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:16:25 -0800
Subject: [PATCH 08/23] (feat) track - api_key in spendLogs

---
 litellm/proxy/_types.py     |  5 ++-
 litellm/proxy/schema.prisma |  3 +-
 litellm/proxy/utils.py      | 82 +++++++++++++++++++++++++++++++++++++
 litellm/utils.py            |  2 +
 schema.prisma               |  5 ++-
 5 files changed, 92 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 9bc6b09b12..21629cb816 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -317,14 +317,15 @@ class LiteLLM_UserTable(LiteLLMBase):
 
 class LiteLLM_SpendLogs(LiteLLMBase):
     request_id: str
+    api_key: str
+    model: Optional[str] = ""
     call_type: str
+    spend: Optional[float] = 0.0
     startTime: Union[str, datetime, None]
     endTime: Union[str, datetime, None]
-    model: Optional[str] = ""
     user: Optional[str] = ""
     modelParameters: Optional[Json] = {}
     messages: Optional[Json] = []
-    spend: Optional[float] = 0.0
     response: Optional[Json] = {}
     usage: Optional[Json] = {}
     metadata: Optional[Json] = {}
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 9049f953d7..2e40a32045 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -36,13 +36,14 @@ model LiteLLM_Config {
 model LiteLLM_SpendLogs {
   request_id          String @unique
   call_type           String
+  api_key             String  @default ("")
+  spend               Float    @default(0.0)
   startTime           DateTime // Assuming start_time is a DateTime field
   endTime             DateTime // Assuming end_time is a DateTime field
   model               String   @default("")
   user                String   @default("")
   modelParameters     Json     @default("{}")// Assuming optional_params is a JSON field
   messages            Json     @default("[]")
-  spend               Float    @default(0.0)
   response            Json     @default("{}")
   usage               Json     @default("{}")
   metadata            Json     @default("{}")
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index f24412a84c..23b66f22d7 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -779,3 +779,85 @@ async def send_email(sender_name, sender_email, receiver_email, subject, html):
 
     except Exception as e:
         print_verbose("An error occurred while sending the email:", str(e))
+
+
+def hash_token(token: str):
+    import hashlib
+
+    # Hash the string using SHA-256
+    hashed_token = hashlib.sha256(token.encode()).hexdigest()
+
+    return hashed_token
+
+
+def get_logging_payload(kwargs, response_obj, start_time, end_time):
+    from litellm.proxy._types import LiteLLM_SpendLogs
+    from pydantic import Json
+    import uuid
+
+    if kwargs == None:
+        kwargs = {}
+    # standardize this function to be used across, s3, dynamoDB, langfuse logging
+    litellm_params = kwargs.get("litellm_params", {})
+    metadata = (
+        litellm_params.get("metadata", {}) or {}
+    )  # if litellm_params['metadata'] == None
+    messages = kwargs.get("messages")
+    optional_params = kwargs.get("optional_params", {})
+    call_type = kwargs.get("call_type", "litellm.completion")
+    cache_hit = kwargs.get("cache_hit", False)
+    usage = response_obj["usage"]
+    id = response_obj.get("id", str(uuid.uuid4()))
+    api_key = metadata.get("user_api_key", "")
+    if api_key is not None and type(api_key) == str:
+        # hash the api_key
+        api_key = hash_token(api_key)
+
+    payload = {
+        "request_id": id,
+        "call_type": call_type,
+        "api_key": api_key,
+        "cache_hit": cache_hit,
+        "startTime": start_time,
+        "endTime": end_time,
+        "model": kwargs.get("model", ""),
+        "user": kwargs.get("user", ""),
+        "modelParameters": optional_params,
+        "messages": messages,
+        "response": response_obj,
+        "usage": usage,
+        "metadata": metadata,
+    }
+
+    json_fields = [
+        field
+        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
+        if field_type == Json or field_type == Optional[Json]
+    ]
+    str_fields = [
+        field
+        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
+        if field_type == str or field_type == Optional[str]
+    ]
+    datetime_fields = [
+        field
+        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
+        if field_type == datetime
+    ]
+
+    for param in json_fields:
+        if param in payload and type(payload[param]) != Json:
+            if type(payload[param]) == litellm.ModelResponse:
+                payload[param] = payload[param].model_dump_json()
+            if type(payload[param]) == litellm.EmbeddingResponse:
+                payload[param] = payload[param].model_dump_json()
+            elif type(payload[param]) == litellm.Usage:
+                payload[param] = payload[param].model_dump_json()
+            else:
+                payload[param] = json.dumps(payload[param])
+
+    for param in str_fields:
+        if param in payload and type(payload[param]) != str:
+            payload[param] = str(payload[param])
+
+    return payload
diff --git a/litellm/utils.py b/litellm/utils.py
index 77f0c331e9..e8740e8aae 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8442,10 +8442,12 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time):
     cache_hit = kwargs.get("cache_hit", False)
     usage = response_obj["usage"]
     id = response_obj.get("id", str(uuid.uuid4()))
+    api_key = metadata.get("user_api_key", "")
 
     payload = {
         "request_id": id,
         "call_type": call_type,
+        "api_key": api_key,
         "cache_hit": cache_hit,
         "startTime": start_time,
         "endTime": end_time,
diff --git a/schema.prisma b/schema.prisma
index a07dcad08e..31eae05c2e 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -35,16 +35,17 @@ model LiteLLM_Config {
 
 model LiteLLM_SpendLogs {
   request_id          String @unique
+  api_key             String  @default ("")
   call_type           String
+  spend               Float    @default(0.0)
   startTime           DateTime // Assuming start_time is a DateTime field
   endTime             DateTime // Assuming end_time is a DateTime field
   model               String   @default("")
   user                String   @default("")
   modelParameters     Json     @default("{}")// Assuming optional_params is a JSON field
   messages            Json     @default("[]")
-  spend               Float    @default(0.0)
   response            Json     @default("{}")
   usage               Json     @default("{}")
   metadata            Json     @default("{}")
   cache_hit           String   @default("")
-}
\ No newline at end of file
+}

From 7bdf0a0a63f2733a38cab7e296cec762af82bfcf Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:21:51 -0800
Subject: [PATCH 09/23] (chore) cleanup utils.py

---
 litellm/utils.py | 69 ------------------------------------------------
 1 file changed, 69 deletions(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index e8740e8aae..f7cc5d2a54 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8423,72 +8423,3 @@ def print_args_passed_to_litellm(original_function, args, kwargs):
     except:
         # This should always be non blocking
         pass
-
-
-def get_logging_payload(kwargs, response_obj, start_time, end_time):
-    from litellm.proxy._types import LiteLLM_SpendLogs
-    from pydantic import Json
-
-    if kwargs == None:
-        kwargs = {}
-    # standardize this function to be used across, s3, dynamoDB, langfuse logging
-    litellm_params = kwargs.get("litellm_params", {})
-    metadata = (
-        litellm_params.get("metadata", {}) or {}
-    )  # if litellm_params['metadata'] == None
-    messages = kwargs.get("messages")
-    optional_params = kwargs.get("optional_params", {})
-    call_type = kwargs.get("call_type", "litellm.completion")
-    cache_hit = kwargs.get("cache_hit", False)
-    usage = response_obj["usage"]
-    id = response_obj.get("id", str(uuid.uuid4()))
-    api_key = metadata.get("user_api_key", "")
-
-    payload = {
-        "request_id": id,
-        "call_type": call_type,
-        "api_key": api_key,
-        "cache_hit": cache_hit,
-        "startTime": start_time,
-        "endTime": end_time,
-        "model": kwargs.get("model", ""),
-        "user": kwargs.get("user", ""),
-        "modelParameters": optional_params,
-        "messages": messages,
-        "response": response_obj,
-        "usage": usage,
-        "metadata": metadata,
-    }
-
-    json_fields = [
-        field
-        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
-        if field_type == Json or field_type == Optional[Json]
-    ]
-    str_fields = [
-        field
-        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
-        if field_type == str or field_type == Optional[str]
-    ]
-    datetime_fields = [
-        field
-        for field, field_type in LiteLLM_SpendLogs.__annotations__.items()
-        if field_type == datetime
-    ]
-
-    for param in json_fields:
-        if param in payload and type(payload[param]) != Json:
-            if type(payload[param]) == ModelResponse:
-                payload[param] = payload[param].model_dump_json()
-            if type(payload[param]) == EmbeddingResponse:
-                payload[param] = payload[param].model_dump_json()
-            elif type(payload[param]) == Usage:
-                payload[param] = payload[param].model_dump_json()
-            else:
-                payload[param] = json.dumps(payload[param])
-
-    for param in str_fields:
-        if param in payload and type(payload[param]) != str:
-            payload[param] = str(payload[param])
-
-    return payload

From 1c987a436eff64147672c071854cd2d7aefd228e Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:34:33 -0800
Subject: [PATCH 10/23] (docs) virtual_keys

---
 docs/my-website/docs/proxy/virtual_keys.md | 35 +++++++++++++++++-----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md
index ee8a990429..3adbb07ff2 100644
--- a/docs/my-website/docs/proxy/virtual_keys.md
+++ b/docs/my-website/docs/proxy/virtual_keys.md
@@ -1,4 +1,4 @@
-# Key Management
+# Virtual Keys
 Track Spend, Set budgets and create virtual keys for the proxy
 
 Grant other's temporary access to your proxy, with keys that expire after a set duration.
@@ -12,7 +12,7 @@ Grant other's temporary access to your proxy, with keys that expire after a set
 
 :::
 
-## Quick Start
+## Setup
 
 Requirements: 
 
@@ -58,16 +58,37 @@ litellm --config /path/to/config.yaml
 curl 'http://0.0.0.0:8000/key/generate' \
 --header 'Authorization: Bearer <your-master-key>' \
 --header 'Content-Type: application/json' \
---data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
+--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai"}}'
 ```
 
+
+## /key/generate
+
+### Request
+```shell
+curl 'http://0.0.0.0:8000/key/generate' \
+--header 'Authorization: Bearer <your-master-key>' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+  "models": ["gpt-3.5-turbo", "gpt-4", "claude-2"],
+  "duration": "20m",
+  "metadata": {"user": "ishaan@berri.ai"},
+  "team_id": "core-infra"
+}'
+```
+
+
+Request Params:
+
 - `models`: *list or null (optional)* - Specify the models a token has access too. If null, then token has access to all models on server. 
 
 - `duration`: *str or null (optional)* Specify the length of time the token is valid for. If null, default is set to 1 hour. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
 
 - `metadata`: *dict or null (optional)* Pass metadata for the created token. If null defaults to {}
 
-Expected response: 
+- `team_id`: *str or null (optional)* Specify team_id for the associated key
+
+### Response
 
 ```python
 {
@@ -76,7 +97,7 @@ Expected response:
 }
 ```
 
-## Keys that don't expire
+### Keys that don't expire
 
 Just set duration to None. 
 
@@ -87,7 +108,7 @@ curl --location 'http://0.0.0.0:8000/key/generate' \
 --data '{"models": ["azure-models"], "aliases": {"mistral-7b": "gpt-3.5-turbo"}, "duration": null}'
 ```
 
-## Upgrade/Downgrade Models 
+### Upgrade/Downgrade Models 
 
 If a user is expected to use a given model (i.e. gpt3-5), and you want to:
 
@@ -137,7 +158,7 @@ curl -X POST "https://0.0.0.0:8000/key/generate" \
 - **How are routing between diff keys/api bases done?** litellm handles this by shuffling between different models in the model list with the same model_name. [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/router.py)
 
 
-## Grant Access to new model 
+### Grant Access to new model 
 
 Use model access groups to give users access to select models, and add new ones to it over time (e.g. mistral, llama-2, etc.)
 

From ea32a8757bd4f984ca0e3cd58c1d9d84f426cad0 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:34:51 -0800
Subject: [PATCH 11/23] (feat) set team_id on virtual_keys

---
 litellm/proxy/_types.py     | 1 +
 litellm/proxy/schema.prisma | 1 +
 schema.prisma               | 1 +
 3 files changed, 3 insertions(+)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index ad90173a44..e258582efb 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -129,6 +129,7 @@ class GenerateKeyRequest(LiteLLMBase):
     config: Optional[dict] = {}
     spend: Optional[float] = 0
     user_id: Optional[str] = None
+    team_id: Optional[str] = None
     max_parallel_requests: Optional[int] = None
     metadata: Optional[dict] = {}
 
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index aa45a88186..1ed76140ed 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -24,6 +24,7 @@ model LiteLLM_VerificationToken {
     aliases    Json  @default("{}")
     config     Json  @default("{}")
     user_id    String?
+    team_id    String?
     max_parallel_requests Int?
     metadata   Json  @default("{}")
 }
diff --git a/schema.prisma b/schema.prisma
index 704ada42c9..3c2bf22cb5 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -24,6 +24,7 @@ model LiteLLM_VerificationToken {
     aliases    Json  @default("{}")
     config     Json  @default("{}")
     user_id    String?
+    team_id    String?
     max_parallel_requests Int?
     metadata   Json  @default("{}")
 }

From 4294657b99ad5939425adf2bfa339c75ffb2b2fa Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:40:48 -0800
Subject: [PATCH 12/23] (fix) use get_logging_payload

---
 litellm/proxy/proxy_server.py               | 3 ++-
 litellm/tests/test_key_generate_dynamodb.py | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 32e985113e..8145d19aa3 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -72,6 +72,7 @@ from litellm.proxy.utils import (
     ProxyLogging,
     _cache_user_row,
     send_email,
+    get_logging_payload,
 )
 from litellm.proxy.secret_managers.google_kms import load_google_kms
 import pydantic
@@ -646,7 +647,7 @@ async def update_database(
         async def _insert_spend_log_to_db():
             # Helper to generate payload to log
             verbose_proxy_logger.debug("inserting spend log to db")
-            payload = litellm.utils.get_logging_payload(
+            payload = get_logging_payload(
                 kwargs=kwargs,
                 response_obj=completion_response,
                 start_time=start_time,
diff --git a/litellm/tests/test_key_generate_dynamodb.py b/litellm/tests/test_key_generate_dynamodb.py
index 09f699af7d..2cfa9c9531 100644
--- a/litellm/tests/test_key_generate_dynamodb.py
+++ b/litellm/tests/test_key_generate_dynamodb.py
@@ -179,6 +179,10 @@ def test_call_with_key_over_budget(custom_db_client):
     # 5. Make a call with a key over budget, expect to fail
     setattr(litellm.proxy.proxy_server, "custom_db_client", custom_db_client)
     setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
+    from litellm._logging import verbose_proxy_logger
+    import logging
+
+    verbose_proxy_logger.setLevel(logging.DEBUG)
     try:
 
         async def test():

From 42ad12b2bd6f0cb0cb19870ef1c2b0ab71eeb696 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:48:52 -0800
Subject: [PATCH 13/23] (fix) support team_id for /key/generate

---
 litellm/proxy/proxy_server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 10c968b1c5..e77223861e 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -1037,6 +1037,7 @@ async def generate_key_helper_fn(
     max_budget: Optional[float] = None,
     token: Optional[str] = None,
     user_id: Optional[str] = None,
+    team_id: Optional[str] = None,
     user_email: Optional[str] = None,
     max_parallel_requests: Optional[int] = None,
     metadata: Optional[dict] = {},

From 90509a159a4518b92ac0f1c0beec0feb7688066b Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:54:08 -0800
Subject: [PATCH 14/23] (fix) write team_id to key table

---
 litellm/proxy/proxy_server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index e77223861e..f951376306 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -1098,6 +1098,7 @@ async def generate_key_helper_fn(
             "config": config_json,
             "spend": spend,
             "user_id": user_id,
+            "team_id": team_id,
             "max_parallel_requests": max_parallel_requests,
             "metadata": metadata_json,
         }

From f405a827e37c5fce5c23094a9b83138d2b6c635d Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 13:55:32 -0800
Subject: [PATCH 15/23] (docs) virtual_keys

---
 docs/my-website/docs/proxy/virtual_keys.md | 32 ++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md
index 3adbb07ff2..49f4ed23c8 100644
--- a/docs/my-website/docs/proxy/virtual_keys.md
+++ b/docs/my-website/docs/proxy/virtual_keys.md
@@ -186,6 +186,38 @@ curl --location 'http://localhost:8000/key/generate' \
 			"max_budget": 0,}'
 ```
 
+<!-- ## /key/update -->
+
+<!-- ### Request
+```shell
+curl 'http://0.0.0.0:8000/key/update' \
+--header 'Authorization: Bearer <your-master-key>' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+  "key": "sk-kdEXbIqZRwEeEiHwdg7sFA",
+  "models": ["gpt-3.5-turbo", "gpt-4", "claude-2"],
+  "metadata": {"user": "ishaan@berri.ai"},
+  "team_id": "core-infra"
+}'
+```
+
+Request Params:
+- key: str - The key that needs to be updated.
+
+- models: list or null (optional) - Specify the models a token has access to. If null, then the token has access to all models on the server.
+
+- duration: str or null (optional) - Specify the length of time the token is valid for. If null, the default is set to 1 hour. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
+
+- metadata: dict or null (optional) - Pass metadata for the updated token. If null, defaults to an empty dictionary.
+
+- team_id: str or null (optional) - Specify the team_id for the associated key.
+
+### Response
+
+```python
+
+``` -->
+
 ## Tracking Spend 
 
 You can get spend for a key by using the `/key/info` endpoint. 

From 5beef6dbcd32271b92d8c12a4e200c90cfc73cde Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 14:33:13 -0800
Subject: [PATCH 16/23] (test) setting team_id

---
 litellm/tests/test_key_generate_prisma.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/litellm/tests/test_key_generate_prisma.py b/litellm/tests/test_key_generate_prisma.py
index 5accd03c65..ae51e4e964 100644
--- a/litellm/tests/test_key_generate_prisma.py
+++ b/litellm/tests/test_key_generate_prisma.py
@@ -543,7 +543,8 @@ def test_generate_and_update_key(prisma_client):
         async def test():
             await litellm.proxy.proxy_server.prisma_client.connect()
             request = NewUserRequest(
-                metadata={"team": "litellm-team3", "project": "litellm-project3"}
+                metadata={"team": "litellm-team3", "project": "litellm-project3"},
+                team_id="litellm-core-infra@gmail.com",
             )
             key = await new_user(request)
             print(key)
@@ -560,6 +561,7 @@ def test_generate_and_update_key(prisma_client):
                 "team": "litellm-team3",
                 "project": "litellm-project3",
             }
+            assert result["info"].team_id == "litellm-core-infra@gmail.com"
 
             request = Request(scope={"type": "http"})
             request._url = URL(url="/update/key")

From 2b6972111e8b3256453b0acdec24c22232e16774 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 14:42:46 -0800
Subject: [PATCH 17/23] (feat) write team_id to User Table

---
 litellm/proxy/proxy_server.py | 3 +++
 litellm/proxy/schema.prisma   | 1 +
 schema.prisma                 | 1 +
 3 files changed, 5 insertions(+)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index f951376306..80e89709d7 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -1081,12 +1081,15 @@ async def generate_key_helper_fn(
     config_json = json.dumps(config)
     metadata_json = json.dumps(metadata)
     user_id = user_id or str(uuid.uuid4())
+    if type(team_id) is not str:
+        team_id = str(team_id)
     try:
         # Create a new verification token (you may want to enhance this logic based on your needs)
         user_data = {
             "max_budget": max_budget,
             "user_email": user_email,
             "user_id": user_id,
+            "team_id": team_id,
             "spend": spend,
             "models": models,
         }
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 1ed76140ed..24f7f4f3d7 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -9,6 +9,7 @@ generator client {
 
 model LiteLLM_UserTable {
 		user_id    String @unique
+    team_id    String?
 		max_budget Float?
     spend      Float    @default(0.0)
     user_email    String?
diff --git a/schema.prisma b/schema.prisma
index 3c2bf22cb5..ae5c53a4df 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -9,6 +9,7 @@ generator client {
 
 model LiteLLM_UserTable {
 		user_id    String @unique
+    team_id    String?
 		max_budget Float?
     spend      Float    @default(0.0)
     user_email    String?

From d5e720e16109a1db7708b92ac49f654397ae7fc6 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 14:45:49 -0800
Subject: [PATCH 18/23] (docs) /key/update

---
 docs/my-website/docs/proxy/virtual_keys.md | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md
index 49f4ed23c8..391f1b60d2 100644
--- a/docs/my-website/docs/proxy/virtual_keys.md
+++ b/docs/my-website/docs/proxy/virtual_keys.md
@@ -186,9 +186,9 @@ curl --location 'http://localhost:8000/key/generate' \
 			"max_budget": 0,}'
 ```
 
-<!-- ## /key/update -->
+## /key/update
 
-<!-- ### Request
+### Request
 ```shell
 curl 'http://0.0.0.0:8000/key/update' \
 --header 'Authorization: Bearer <your-master-key>' \
@@ -206,8 +206,6 @@ Request Params:
 
 - models: list or null (optional) - Specify the models a token has access to. If null, then the token has access to all models on the server.
 
-- duration: str or null (optional) - Specify the length of time the token is valid for. If null, the default is set to 1 hour. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
-
 - metadata: dict or null (optional) - Pass metadata for the updated token. If null, defaults to an empty dictionary.
 
 - team_id: str or null (optional) - Specify the team_id for the associated key.
@@ -215,8 +213,15 @@ Request Params:
 ### Response
 
 ```python
+{
+  "key": "sk-kdEXbIqZRwEeEiHwdg7sFA",
+  "models": ["gpt-3.5-turbo", "gpt-4", "claude-2"],
+  "metadata": {
+    "user": "ishaan@berri.ai"
+  }
+}
 
-``` -->
+```
 
 ## Tracking Spend 
 

From 0e3e8050d7160f3c39565982a91d77d3bcfa2e75 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 14:54:11 -0800
Subject: [PATCH 19/23] (docs) /key/info

---
 docs/my-website/docs/proxy/virtual_keys.md | 36 +++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md
index 391f1b60d2..e9ce490786 100644
--- a/docs/my-website/docs/proxy/virtual_keys.md
+++ b/docs/my-website/docs/proxy/virtual_keys.md
@@ -186,6 +186,40 @@ curl --location 'http://localhost:8000/key/generate' \
 			"max_budget": 0,}'
 ```
 
+
+## /key/info
+
+### Request
+```shell
+curl -X GET "http://0.0.0.0:8000/key/info?key=sk-02Wr4IAlN3NvPXvL5JVvDA" \
+-H "Authorization: Bearer sk-1234"
+```
+
+Request Params:
+- key: str - The key you want the info for
+
+### Response
+
+```json
+{
+  "key": "sk-02Wr4IAlN3NvPXvL5JVvDA",
+  "info": {
+    "token": "80321a12d03412c527f2bd9db5fabd746abead2e1d50b435a534432fbaca9ef5",
+    "spend": 0.0,
+    "expires": "2024-01-18T23:52:09.125000+00:00",
+    "models": ["azure-gpt-3.5", "azure-embedding-model"],
+    "aliases": {},
+    "config": {},
+    "user_id": "ishaan2@berri.ai",
+    "team_id": "None",
+    "max_parallel_requests": null,
+    "metadata": {}
+  }
+}
+
+
+```
+
 ## /key/update
 
 ### Request
@@ -212,7 +246,7 @@ Request Params:
 
 ### Response
 
-```python
+```json
 {
   "key": "sk-kdEXbIqZRwEeEiHwdg7sFA",
   "models": ["gpt-3.5-turbo", "gpt-4", "claude-2"],

From cdede8836fc11245103f384afa64fa1053c58d33 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 15:16:40 -0800
Subject: [PATCH 20/23] (docs)virtual keys

---
 docs/my-website/docs/proxy/virtual_keys.md | 25 ++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md
index e9ce490786..1cb28a2e3d 100644
--- a/docs/my-website/docs/proxy/virtual_keys.md
+++ b/docs/my-website/docs/proxy/virtual_keys.md
@@ -200,6 +200,7 @@ Request Params:
 
 ### Response
 
+`token` is the hashed key (The DB stores the hashed key for security)
 ```json
 {
   "key": "sk-02Wr4IAlN3NvPXvL5JVvDA",
@@ -257,6 +258,30 @@ Request Params:
 
 ```
 
+
+## /key/delete
+
+### Request
+```shell
+curl 'http://0.0.0.0:8000/key/delete' \
+--header 'Authorization: Bearer <your-master-key>' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+  "keys": ["sk-kdEXbIqZRwEeEiHwdg7sFA"]
+}'
+```
+
+Request Params:
+- keys: List[str] - List of keys to delete
+
+### Response
+
+```json
+{
+  "deleted_keys": ["sk-kdEXbIqZRwEeEiHwdg7sFA"]
+}
+```
+
 ## Tracking Spend 
 
 You can get spend for a key by using the `/key/info` endpoint. 

From 340706565fc4a02c3a48a0bc1d06d30c07417d6b Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 18 Jan 2024 15:23:05 -0800
Subject: [PATCH 21/23] (fix) add team_id to doc string

---
 litellm/proxy/proxy_server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 80e89709d7..6a6a59ee11 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -2048,6 +2048,7 @@ async def generate_key_fn(
 
     Parameters:
     - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). **(Default is set to 1 hour.)**
+    - team_id: Optional[str] - The team id of the user
     - models: Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models)
     - aliases: Optional[dict] - Any alias mappings, on top of anything in the config.yaml model list. - https://docs.litellm.ai/docs/proxy/virtual_keys#managing-auth---upgradedowngrade-models
     - config: Optional[dict] - any key-specific configs, overrides config in config.yaml

From e0aaa94f28304e362dabd39b164940834ac0fa50 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Thu, 18 Jan 2024 17:13:54 -0800
Subject: [PATCH 22/23] fix(main.py): read azure ad token from optional params
 extra body

---
 litellm/main.py                  | 6 +++---
 litellm/tests/test_completion.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/litellm/main.py b/litellm/main.py
index 6320858167..e8022f464f 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -692,9 +692,9 @@ def completion(
                 or get_secret("AZURE_API_KEY")
             )
 
-            azure_ad_token = optional_params.pop("azure_ad_token", None) or get_secret(
-                "AZURE_AD_TOKEN"
-            )
+            azure_ad_token = optional_params.get("extra_body", {}).pop(
+                "azure_ad_token", None
+            ) or get_secret("AZURE_AD_TOKEN")
 
             headers = headers or litellm.headers
 
diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 71884bde73..ab8673bca1 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -268,7 +268,7 @@ def test_completion_azure_gpt4_vision():
         pytest.fail(f"Error occurred: {e}")
 
 
-test_completion_azure_gpt4_vision()
+# test_completion_azure_gpt4_vision()
 
 
 @pytest.mark.skip(reason="this test is flaky")
@@ -990,9 +990,9 @@ def test_azure_openai_ad_token():
         print("azure ad token respoonse\n")
         print(response)
         litellm.input_callback = []
-    except:
+    except Exception as e:
         litellm.input_callback = []
-        pass
+        pytest.fail(f"An exception occurs - {str(e)}")
 
 
 # test_azure_openai_ad_token()

From b21a357cedb9b5928f42a733ab7e9e2097e5926a Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Thu, 18 Jan 2024 17:14:22 -0800
Subject: [PATCH 23/23] =?UTF-8?q?bump:=20version=201.18.2=20=E2=86=92=201.?=
 =?UTF-8?q?18.3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index cfef03eea8..5941234180 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.18.2"
+version = "1.18.3"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"
@@ -61,7 +61,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.18.2"
+version = "1.18.3"
 version_files = [
     "pyproject.toml:^version"
 ]