phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #2945 from ushuz/hotfix

Browse source 
Fix image generation pre call hook call type
This commit is contained in:
Krish Dholakia 2024-04-10 21:53:00 -07:00 • committed by GitHub
commit 6749306239
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 44 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

92
litellm/proxy/proxy_server.py
View file

@ -352,9 +352,9 @@ async def user_api_key_auth(
### LITELLM-DEFINED AUTH FUNCTION ### ### LITELLM-DEFINED AUTH FUNCTION ###
#### IF JWT #### #### IF JWT ####
""" """
LiteLLM supports using JWTs. LiteLLM supports using JWTs.
Enable this in proxy config, by setting Enable this in proxy config, by setting
``` ```
general_settings: general_settings:
enable_jwt_auth: true enable_jwt_auth: true
@ -3255,8 +3255,6 @@ async def completion(
user_api_key_dict=user_api_key_dict, data=data, call_type="completion" user_api_key_dict=user_api_key_dict, data=data, call_type="completion"
) )
start_time = time.time()
### ROUTE THE REQUESTs ### ### ROUTE THE REQUESTs ###
router_model_names = llm_router.model_names if llm_router is not None else [] router_model_names = llm_router.model_names if llm_router is not None else []
# skip router if user passed their key # skip router if user passed their key
@ -3853,11 +3851,9 @@ async def image_generation(
### CALL HOOKS ### - modify incoming data / reject request before calling the model ### CALL HOOKS ### - modify incoming data / reject request before calling the model
data = await proxy_logging_obj.pre_call_hook( data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict, data=data, call_type="embeddings" user_api_key_dict=user_api_key_dict, data=data, call_type="image_generation"
) )
start_time = time.time()
## ROUTE TO CORRECT ENDPOINT ## ## ROUTE TO CORRECT ENDPOINT ##
# skip router if user passed their key # skip router if user passed their key
if "api_key" in data: if "api_key" in data:
@ -4102,7 +4098,7 @@ async def moderations(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
): ):
""" """
The moderations endpoint is a tool you can use to check whether content complies with an LLM Providers policies. The moderations endpoint is a tool you can use to check whether content complies with an LLM Providers policies.
Quick Start Quick Start
``` ```
@ -4274,7 +4270,7 @@ async def generate_key_fn(
- permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false} - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
- model_max_budget: Optional[dict] - key-specific model budget in USD. Example - {"text-davinci-002": 0.5, "gpt-3.5-turbo": 0.5}. IF null or {} then no model specific budget. - model_max_budget: Optional[dict] - key-specific model budget in USD. Example - {"text-davinci-002": 0.5, "gpt-3.5-turbo": 0.5}. IF null or {} then no model specific budget.
Examples: Examples:
1. Allow users to turn on/off pii masking 1. Allow users to turn on/off pii masking
@ -4579,7 +4575,7 @@ async def info_key_fn_v2(
user_api_key_dict: UserAPIKeyAuth = Dependency representing the user's API key user_api_key_dict: UserAPIKeyAuth = Dependency representing the user's API key
Returns: Returns:
Dict containing the key and its associated information Dict containing the key and its associated information
Example Curl: Example Curl:
``` ```
curl -X GET "http://0.0.0.0:8000/key/info" \ curl -X GET "http://0.0.0.0:8000/key/info" \
@ -4646,7 +4642,7 @@ async def info_key_fn(
user_api_key_dict: UserAPIKeyAuth = Dependency representing the user's API key user_api_key_dict: UserAPIKeyAuth = Dependency representing the user's API key
Returns: Returns:
Dict containing the key and its associated information Dict containing the key and its associated information
Example Curl: Example Curl:
``` ```
curl -X GET "http://0.0.0.0:8000/key/info?key=sk-02Wr4IAlN3NvPXvL5JVvDA" \ curl -X GET "http://0.0.0.0:8000/key/info?key=sk-02Wr4IAlN3NvPXvL5JVvDA" \
@ -4706,7 +4702,7 @@ async def spend_key_fn():
""" """
View all keys created, ordered by spend View all keys created, ordered by spend
Example Request: Example Request:
``` ```
curl -X GET "http://0.0.0.0:8000/spend/keys" \ curl -X GET "http://0.0.0.0:8000/spend/keys" \
-H "Authorization: Bearer sk-1234" -H "Authorization: Bearer sk-1234"
@ -4743,7 +4739,7 @@ async def spend_user_fn(
""" """
View all users created, ordered by spend View all users created, ordered by spend
Example Request: Example Request:
``` ```
curl -X GET "http://0.0.0.0:8000/spend/users" \ curl -X GET "http://0.0.0.0:8000/spend/users" \
-H "Authorization: Bearer sk-1234" -H "Authorization: Bearer sk-1234"
@ -6100,21 +6096,21 @@ async def new_team(
Returns: Returns:
- team_id: (str) Unique team id - used for tracking spend across multiple keys for same team id. - team_id: (str) Unique team id - used for tracking spend across multiple keys for same team id.
_deprecated_params: _deprecated_params:
- admins: list - A list of user_id's for the admin role - admins: list - A list of user_id's for the admin role
- users: list - A list of user_id's for the user role - users: list - A list of user_id's for the user role
Example Request: Example Request:
``` ```
curl --location 'http://0.0.0.0:8000/team/new' \ curl --location 'http://0.0.0.0:8000/team/new' \
--header 'Authorization: Bearer sk-1234' \ --header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data '{ --data '{
"team_alias": "my-new-team_2", "team_alias": "my-new-team_2",
"members_with_roles": [{"role": "admin", "user_id": "user-1234"}, "members_with_roles": [{"role": "admin", "user_id": "user-1234"},
{"role": "user", "user_id": "user-2434"}] {"role": "user", "user_id": "user-2434"}]
}' }'
@ -6242,7 +6238,7 @@ async def update_team(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
): ):
""" """
[RECOMMENDED] - use `/team/member_add` to add new team members instead [RECOMMENDED] - use `/team/member_add` to add new team members instead
You can now update team budget / rate limits via /team/update You can now update team budget / rate limits via /team/update
@ -6255,16 +6251,16 @@ async def update_team(
- max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
- models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed. - models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed.
- blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id. - blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id.
Example - update team TPM Limit Example - update team TPM Limit
``` ```
curl --location 'http://0.0.0.0:8000/team/update' \ curl --location 'http://0.0.0.0:8000/team/update' \
--header 'Authorization: Bearer sk-1234' \ --header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data-raw '{ --data-raw '{
"team_id": "litellm-test-client-id-new", "team_id": "litellm-test-client-id-new",
"tpm_limit": 100 "tpm_limit": 100
@ -6358,7 +6354,7 @@ async def team_member_add(
data: TeamMemberAddRequest, data: TeamMemberAddRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
): ):
""" """
[BETA] [BETA]
Add new members (either via user_email or user_id) to a team Add new members (either via user_email or user_id) to a team
@ -6367,11 +6363,11 @@ async def team_member_add(
``` ```
curl -X POST 'http://0.0.0.0:8000/team/update' \ curl -X POST 'http://0.0.0.0:8000/team/update' \
-H 'Authorization: Bearer sk-1234' \ -H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \ -H 'Content-Type: application/json' \
-D '{ -D '{
"team_id": "45e3e396-ee08-4a61-a88e-16b3ce7e0849", "team_id": "45e3e396-ee08-4a61-a88e-16b3ce7e0849",
"member": {"role": "user", "user_id": "krrish247652@berri.ai"} "member": {"role": "user", "user_id": "krrish247652@berri.ai"}
@ -6451,7 +6447,7 @@ async def team_member_delete(
data: TeamMemberDeleteRequest, data: TeamMemberDeleteRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
): ):
""" """
[BETA] [BETA]
delete members (either via user_email or user_id) from a team delete members (either via user_email or user_id) from a team
@ -6459,11 +6455,11 @@ async def team_member_delete(
If user doesn't exist, an exception will be raised If user doesn't exist, an exception will be raised
``` ```
curl -X POST 'http://0.0.0.0:8000/team/update' \ curl -X POST 'http://0.0.0.0:8000/team/update' \
-H 'Authorization: Bearer sk-1234' \ -H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \ -H 'Content-Type: application/json' \
-D '{ -D '{
"team_id": "45e3e396-ee08-4a61-a88e-16b3ce7e0849", "team_id": "45e3e396-ee08-4a61-a88e-16b3ce7e0849",
"user_id": "krrish247652@berri.ai" "user_id": "krrish247652@berri.ai"
@ -6565,11 +6561,11 @@ async def delete_team(
``` ```
curl --location 'http://0.0.0.0:8000/team/delete' \ curl --location 'http://0.0.0.0:8000/team/delete' \
--header 'Authorization: Bearer sk-1234' \ --header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data-raw '{ --data-raw '{
"team_ids": ["45e3e396-ee08-4a61-a88e-16b3ce7e0849"] "team_ids": ["45e3e396-ee08-4a61-a88e-16b3ce7e0849"]
}' }'
@ -6737,27 +6733,27 @@ async def new_organization(
Only admins can create orgs. Only admins can create orgs.
# Parameters # Parameters
- `organization_alias`: *str* = The name of the organization. - `organization_alias`: *str* = The name of the organization.
- `models`: *List* = The models the organization has access to. - `models`: *List* = The models the organization has access to.
- `budget_id`: *Optional[str]* = The id for a budget (tpm/rpm/max budget) for the organization. - `budget_id`: *Optional[str]* = The id for a budget (tpm/rpm/max budget) for the organization.
### IF NO BUDGET ID - CREATE ONE WITH THESE PARAMS ### ### IF NO BUDGET ID - CREATE ONE WITH THESE PARAMS ###
- `max_budget`: *Optional[float]* = Max budget for org - `max_budget`: *Optional[float]* = Max budget for org
- `tpm_limit`: *Optional[int]* = Max tpm limit for org - `tpm_limit`: *Optional[int]* = Max tpm limit for org
- `rpm_limit`: *Optional[int]* = Max rpm limit for org - `rpm_limit`: *Optional[int]* = Max rpm limit for org
- `model_max_budget`: *Optional[dict]* = Max budget for a specific model - `model_max_budget`: *Optional[dict]* = Max budget for a specific model
- `budget_duration`: *Optional[str]* = Frequency of reseting org budget - `budget_duration`: *Optional[str]* = Frequency of reseting org budget
Case 1: Create new org **without** a budget_id Case 1: Create new org **without** a budget_id
```bash ```bash
curl --location 'http://0.0.0.0:4000/organization/new' \ curl --location 'http://0.0.0.0:4000/organization/new' \
--header 'Authorization: Bearer sk-1234' \ --header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data '{ --data '{
"organization_alias": "my-secret-org", "organization_alias": "my-secret-org",
"models": ["model1", "model2"], "models": ["model1", "model2"],
@ -6767,15 +6763,15 @@ async def new_organization(
``` ```
Case 2: Create new org **with** a budget_id Case 2: Create new org **with** a budget_id
```bash ```bash
curl --location 'http://0.0.0.0:4000/organization/new' \ curl --location 'http://0.0.0.0:4000/organization/new' \
--header 'Authorization: Bearer sk-1234' \ --header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data '{ --data '{
"organization_alias": "my-secret-org", "organization_alias": "my-secret-org",
"models": ["model1", "model2"], "models": ["model1", "model2"],
@ -7352,7 +7348,7 @@ async def async_queue_request(
): ):
global general_settings, user_debug, proxy_logging_obj global general_settings, user_debug, proxy_logging_obj
""" """
v2 attempt at a background worker to handle queuing. v2 attempt at a background worker to handle queuing.
Just supports /chat/completion calls currently. Just supports /chat/completion calls currently.