From 9b8c14360140d4b877d54977d577738876a1fc7c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:17:46 -0700 Subject: [PATCH 1/6] feat - rename end_user -> customer --- litellm/proxy/proxy_server.py | 83 +++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bdb5edba..b7985ce5b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7131,13 +7131,15 @@ async def global_predict_spend_logs(request: Request): #### INTERNAL USER MANAGEMENT #### @router.post( "/user/new", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], response_model=NewUserResponse, ) async def new_user(data: NewUserRequest): """ - Use this to create a new user with a budget. This creates a new user and generates a new api key for the new user. The new api key is returned. + Use this to create a new INTERNAL user with a budget. + Internal Users can access LiteLLM Admin UI to make keys, request access to models. + This creates a new user and generates a new api key for the new user. The new api key is returned. Returns user id, budget + new key. @@ -7208,7 +7210,9 @@ async def new_user(data: NewUserRequest): @router.post( - "/user/auth", tags=["user management"], dependencies=[Depends(user_api_key_auth)] + "/user/auth", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], ) async def user_auth(request: Request): """ @@ -7274,7 +7278,9 @@ async def user_auth(request: Request): @router.get( - "/user/info", tags=["user management"], dependencies=[Depends(user_api_key_auth)] + "/user/info", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], ) async def user_info( user_id: Optional[str] = fastapi.Query( @@ -7446,7 +7452,9 @@ async def user_info( @router.post( - "/user/update", tags=["user management"], dependencies=[Depends(user_api_key_auth)] + "/user/update", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], ) async def user_update(data: UpdateUserRequest): """ @@ -7540,7 +7548,7 @@ async def user_update(data: UpdateUserRequest): @router.post( "/user/request_model", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) async def user_request_model(request: Request): @@ -7593,7 +7601,7 @@ async def user_request_model(request: Request): @router.get( "/user/get_requests", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) async def user_get_requests(): @@ -7635,7 +7643,7 @@ async def user_get_requests(): @router.get( "/user/get_users", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) async def get_users( @@ -7672,7 +7680,13 @@ async def get_users( @router.post( "/end_user/block", - tags=["End User Management"], + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, +) +@router.post( + "/customer/block", + tags=["Customer Management"], dependencies=[Depends(user_api_key_auth)], ) async def block_user(data: BlockUsers): @@ -7715,9 +7729,15 @@ async def block_user(data: BlockUsers): @router.post( "/end_user/unblock", - tags=["End User Management"], + tags=["Customer Management"], dependencies=[Depends(user_api_key_auth)], ) +@router.post( + "/customer/unblock", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, +) async def unblock_user(data: BlockUsers): """ [BETA] Unblock calls with this user id @@ -7762,7 +7782,13 @@ async def unblock_user(data: BlockUsers): @router.post( "/end_user/new", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], +) +@router.post( + "/customer/new", + tags=["Customer Management"], dependencies=[Depends(user_api_key_auth)], ) async def new_end_user( @@ -7779,18 +7805,13 @@ async def new_end_user( Example curl: ``` - curl --location 'http://0.0.0.0:4000/end_user/new' \ + curl --location 'http://0.0.0.0:4000/customer/new' \ --header 'Authorization: Bearer sk-1234' \ --header 'Content-Type: application/json' \ --data '{ - "end_user_id" : "ishaan-jaff-3", <- specific customer - - "allowed_region": "eu" <- set region for models - - + - + "user_id" : "ishaan-jaff-3", + "allowed_region": "eu" "default_model": "azure/gpt-3.5-turbo-eu" <- all calls from this user, use this model? - }' # return end-user object @@ -7860,9 +7881,15 @@ async def new_end_user( return end_user_record +@router.get( + "/customer/info", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], +) @router.get( "/end_user/info", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) async def end_user_info( @@ -7885,9 +7912,15 @@ async def end_user_info( return user_info +@router.post( + "/customer/update", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], +) @router.post( "/end_user/update", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) async def update_end_user(): @@ -7897,9 +7930,15 @@ async def update_end_user(): pass +@router.post( + "/customer/delete", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], +) @router.post( "/end_user/delete", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) async def delete_end_user(): From 8792b8c7fa92b51cdd02d89621cba237c8296511 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:18:58 -0700 Subject: [PATCH 2/6] docs - rename end user -> customer --- docs/my-website/docs/proxy/users.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 556ae7f92..ec2be9cdc 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -13,7 +13,7 @@ Requirements: You can set budgets at 3 levels: - For the proxy - For an internal user -- For an end-user +- For a customer (end-user) - For a key - For a key (model specific budgets) @@ -173,7 +173,7 @@ curl --location 'http://localhost:4000/chat/completions' \ ``` - + Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user** @@ -452,7 +452,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ ``` - + :::info @@ -477,12 +477,12 @@ curl --location 'http://0.0.0.0:4000/budget/new' \ ``` -#### Step 2. Create `End-User` with Budget +#### Step 2. Create `Customer` with Budget -We use `budget_id="free-tier"` from Step 1 when creating this new end user +We use `budget_id="free-tier"` from Step 1 when creating this new customers ```shell -curl --location 'http://0.0.0.0:4000/end_user/new' \ +curl --location 'http://0.0.0.0:4000/customer/new' \ --header 'Authorization: Bearer sk-1234' \ --header 'Content-Type: application/json' \ --data '{ @@ -492,7 +492,7 @@ curl --location 'http://0.0.0.0:4000/end_user/new' \ ``` -#### Step 3. Pass end user id in `/chat/completions` requests +#### Step 3. Pass `user_id` id in `/chat/completions` requests Pass the `user_id` from Step 2 as `user="palantir"` From cdf32ebf0ec3ae66eacc317538a1d5aa5a70a1a1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:20:44 -0700 Subject: [PATCH 3/6] docs string - > end user /new --- litellm/proxy/proxy_server.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b7985ce5b..ea1677aa3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7796,9 +7796,8 @@ async def new_end_user( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - [TODO] Needs to be implemented. - - Allow creating a new end-user + Allow creating a new Customer + NOTE: This used to be called `/end_user/new`, we will still be maintaining compatibility for /end_user/XXX for these endpoints - Allow specifying allowed regions - Allow specifying default model From 24f0b82755b1262c663157d60642827a669cf5ef Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:29:09 -0700 Subject: [PATCH 4/6] feat - add validation for existing customers --- litellm/proxy/proxy_server.py | 102 +++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ea1677aa3..b22bbd291 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7833,51 +7833,77 @@ async def new_end_user( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) + try: - ## VALIDATION ## - if data.default_model is not None: - if llm_router is None: - raise HTTPException( - status_code=422, detail={"error": CommonProxyErrors.no_llm_router.value} - ) - elif data.default_model not in llm_router.get_model_names(): - raise HTTPException( - status_code=422, - detail={ - "error": "Default Model not on proxy. Configure via `/model/new` or config.yaml. Default_model={}, proxy_model_names={}".format( - data.default_model, set(llm_router.get_model_names()) - ) - }, + ## VALIDATION ## + if data.default_model is not None: + if llm_router is None: + raise HTTPException( + status_code=422, + detail={"error": CommonProxyErrors.no_llm_router.value}, + ) + elif data.default_model not in llm_router.get_model_names(): + raise HTTPException( + status_code=422, + detail={ + "error": "Default Model not on proxy. Configure via `/model/new` or config.yaml. Default_model={}, proxy_model_names={}".format( + data.default_model, set(llm_router.get_model_names()) + ) + }, + ) + + new_end_user_obj: Dict = {} + + ## CREATE BUDGET ## if set + if data.max_budget is not None: + budget_record = await prisma_client.db.litellm_budgettable.create( + data={ + "max_budget": data.max_budget, + "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, # type: ignore + "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, + } ) - new_end_user_obj: Dict = {} + new_end_user_obj["budget_id"] = budget_record.budget_id + elif data.budget_id is not None: + new_end_user_obj["budget_id"] = data.budget_id - ## CREATE BUDGET ## if set - if data.max_budget is not None: - budget_record = await prisma_client.db.litellm_budgettable.create( - data={ - "max_budget": data.max_budget, - "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, # type: ignore - "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, - } + _user_data = data.dict(exclude_none=True) + + for k, v in _user_data.items(): + if k != "max_budget" and k != "budget_id": + new_end_user_obj[k] = v + + ## WRITE TO DB ## + end_user_record = await prisma_client.db.litellm_endusertable.create( + data=new_end_user_obj # type: ignore ) - new_end_user_obj["budget_id"] = budget_record.budget_id - elif data.budget_id is not None: - new_end_user_obj["budget_id"] = data.budget_id + return end_user_record + except Exception as e: + if "Unique constraint failed on the fields: (`user_id`)" in str(e): + raise ProxyException( + message=f"Customer already exists, passed user_id={data.user_id}. Please pass a new user_id.", + type="bad_request", + code=400, + param="user_id", + ) - _user_data = data.dict(exclude_none=True) - - for k, v in _user_data.items(): - if k != "max_budget" and k != "budget_id": - new_end_user_obj[k] = v - - ## WRITE TO DB ## - end_user_record = await prisma_client.db.litellm_endusertable.create( - data=new_end_user_obj # type: ignore - ) - - return end_user_record + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) @router.get( From f5886164988f3baa3e33f285bd8efb7c7147b050 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:02:20 -0700 Subject: [PATCH 5/6] fix - /customer/update --- litellm/proxy/_types.py | 14 +++++++ litellm/proxy/proxy_server.py | 77 +++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e85f116f7..d93c1e33b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -540,6 +540,20 @@ class NewEndUserRequest(LiteLLMBase): return values +class UpdateEndUserRequest(LiteLLMBase): + user_id: str + alias: Optional[str] = None # human-friendly alias + blocked: bool = False # allow/disallow requests for this end-user + max_budget: Optional[float] = None + budget_id: Optional[str] = None # give either a budget_id or max_budget + allowed_model_region: Optional[Literal["eu"]] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) + + class Member(LiteLLMBase): role: Literal["admin", "user"] user_id: Optional[str] = None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b22bbd291..66a1d5e16 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7809,7 +7809,8 @@ async def new_end_user( --header 'Content-Type: application/json' \ --data '{ "user_id" : "ishaan-jaff-3", - "allowed_region": "eu" + "allowed_region": "eu", + "budget_id": "free_tier", "default_model": "azure/gpt-3.5-turbo-eu" <- all calls from this user, use this model? }' @@ -7948,10 +7949,80 @@ async def end_user_info( include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) -async def update_end_user(): +async def update_end_user( + data: UpdateEndUserRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ - [TODO] Needs to be implemented. + Example curl + + ``` + curl --location 'http://0.0.0.0:4000/customer/update' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_id": "test-litellm-user-4", + "budget_id": "paid_tier" + }' + + See below for all params + ``` """ + + global prisma_client + try: + data_json: dict = data.json() + # get the row from db + if prisma_client is None: + raise Exception("Not connected to DB!") + + # get non default values for key + non_default_values = {} + for k, v in data_json.items(): + if v is not None and v not in ( + [], + {}, + 0, + ): # models default to [], spend defaults to 0, we should not reset these values + non_default_values[k] = v + + ## ADD USER, IF NEW ## + verbose_proxy_logger.debug("/customer/update: Received data = %s", data) + if data.user_id is not None and len(data.user_id) > 0: + non_default_values["user_id"] = data.user_id # type: ignore + verbose_proxy_logger.debug("In update customer, user_id condition block.") + response = await prisma_client.db.litellm_endusertable.update( + where={"user_id": data.user_id}, data=non_default_values # type: ignore + ) + if response is None: + raise ValueError( + f"Failed updating customer data. User ID does not exist passed user_id={data.user_id}" + ) + verbose_proxy_logger.debug( + f"received response from updating prisma client. response={response}" + ) + return response + else: + raise ValueError(f"user_id is required, passed user_id = {data.user_id}") + + # update based on remaining passed in values + except Exception as e: + traceback.print_exc() + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) pass From 0feeb53868051cd0e3424dbbcdc476663dc9c4d8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:24:20 -0700 Subject: [PATCH 6/6] fix - working customer/delete --- litellm/proxy/_types.py | 21 +++++++++- litellm/proxy/proxy_server.py | 74 +++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d93c1e33b..13d83cbc2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -519,7 +519,11 @@ class UpdateUserRequest(GenerateRequestBase): return values -class NewEndUserRequest(LiteLLMBase): +class NewCustomerRequest(LiteLLMBase): + """ + Create a new customer, allocate a budget to them + """ + user_id: str alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user @@ -540,7 +544,12 @@ class NewEndUserRequest(LiteLLMBase): return values -class UpdateEndUserRequest(LiteLLMBase): +class UpdateCustomerRequest(LiteLLMBase): + """ + Update a Customer, use this to update customer budgets etc + + """ + user_id: str alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user @@ -554,6 +563,14 @@ class UpdateEndUserRequest(LiteLLMBase): ) +class DeleteCustomerRequest(LiteLLMBase): + """ + Delete multiple Customers + """ + + user_ids: List[str] + + class Member(LiteLLMBase): role: Literal["admin", "user"] user_id: Optional[str] = None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 66a1d5e16..0778e678a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7792,7 +7792,7 @@ async def unblock_user(data: BlockUsers): dependencies=[Depends(user_api_key_auth)], ) async def new_end_user( - data: NewEndUserRequest, + data: NewCustomerRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -7950,7 +7950,7 @@ async def end_user_info( dependencies=[Depends(user_api_key_auth)], ) async def update_end_user( - data: UpdateEndUserRequest, + data: UpdateCustomerRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -8037,10 +8037,76 @@ async def update_end_user( include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) -async def delete_end_user(): +async def delete_end_user( + data: DeleteCustomerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ - [TODO] Needs to be implemented. + Example curl + + ``` + curl --location 'http://0.0.0.0:4000/customer/delete' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_ids" :["ishaan-jaff-5"] + }' + + See below for all params + ``` """ + global prisma_client + + try: + if prisma_client is None: + raise Exception("Not connected to DB!") + + verbose_proxy_logger.debug("/customer/delete: Received data = %s", data) + if ( + data.user_ids is not None + and isinstance(data.user_ids, list) + and len(data.user_ids) > 0 + ): + response = await prisma_client.db.litellm_endusertable.delete_many( + where={"user_id": {"in": data.user_ids}} + ) + if response is None: + raise ValueError( + f"Failed deleting customer data. User ID does not exist passed user_id={data.user_ids}" + ) + if response != len(data.user_ids): + raise ValueError( + f"Failed deleting all customer data. User ID does not exist passed user_id={data.user_ids}. Deleted {response} customers, passed {len(data.user_ids)} customers" + ) + verbose_proxy_logger.debug( + f"received response from updating prisma client. response={response}" + ) + return { + "deleted_customers": response, + "message": "Successfully deleted customers with ids: " + + str(data.user_ids), + } + else: + raise ValueError(f"user_id is required, passed user_id = {data.user_ids}") + + # update based on remaining passed in values + except Exception as e: + traceback.print_exc() + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) pass