phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

use async functions

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-26 14:31:59 -07:00
parent 16c0307eab
commit d71b86cd24
2 changed files with 30 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/model_prices_and_context_window_backup.json
View file

@ -2454,6 +2454,17 @@
"mode": "chat", "mode": "chat",
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models" "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models"
}, },
"vertex_ai/meta/llama-3.2-90b-vision-instruct-maas": {
"max_tokens": 8192,
"max_input_tokens": 128000,
"max_output_tokens": 8192,
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"litellm_provider": "vertex_ai-llama_models",
"mode": "chat",
"supports_system_messages": true,
"source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas"
},
"vertex_ai/mistral-large@latest": { "vertex_ai/mistral-large@latest": {
"max_tokens": 8191, "max_tokens": 8191,
"max_input_tokens": 128000, "max_input_tokens": 128000,

29
litellm/proxy/proxy_server.py
View file

@ -901,9 +901,6 @@ async def update_database(
- Update litellm-proxy-budget row (global proxy spend) - Update litellm-proxy-budget row (global proxy spend)
""" """
## if an end-user is passed in, do an upsert - we can't guarantee they already exist in db ## if an end-user is passed in, do an upsert - we can't guarantee they already exist in db
existing_token_obj = await user_api_key_cache.async_get_cache(
key=hashed_token
)
existing_user_obj = await user_api_key_cache.async_get_cache(key=user_id) existing_user_obj = await user_api_key_cache.async_get_cache(key=user_id)
if existing_user_obj is not None and isinstance(existing_user_obj, dict): if existing_user_obj is not None and isinstance(existing_user_obj, dict):
existing_user_obj = LiteLLM_UserTable(**existing_user_obj) existing_user_obj = LiteLLM_UserTable(**existing_user_obj)
@ -1156,7 +1153,9 @@ async def update_cache(
# Update the cost column for the given token # Update the cost column for the given token
existing_spend_obj.spend = new_spend existing_spend_obj.spend = new_spend
user_api_key_cache.set_cache(key=hashed_token, value=existing_spend_obj) await user_api_key_cache.async_set_cache(
key=hashed_token, value=existing_spend_obj
)
### UPDATE USER SPEND ### ### UPDATE USER SPEND ###
async def _update_user_cache(): async def _update_user_cache():
@ -1185,10 +1184,12 @@ async def update_cache(
# Update the cost column for the given user # Update the cost column for the given user
if isinstance(existing_spend_obj, dict): if isinstance(existing_spend_obj, dict):
existing_spend_obj["spend"] = new_spend existing_spend_obj["spend"] = new_spend
user_api_key_cache.set_cache(key=_id, value=existing_spend_obj) await user_api_key_cache.async_set_cache(
key=_id, value=existing_spend_obj
)
else: else:
existing_spend_obj.spend = new_spend existing_spend_obj.spend = new_spend
user_api_key_cache.set_cache( await user_api_key_cache.async_set_cache(
key=_id, value=existing_spend_obj.json() key=_id, value=existing_spend_obj.json()
) )
## UPDATE GLOBAL PROXY ## ## UPDATE GLOBAL PROXY ##
@ -1237,10 +1238,14 @@ async def update_cache(
# Update the cost column for the given user # Update the cost column for the given user
if isinstance(existing_spend_obj, dict): if isinstance(existing_spend_obj, dict):
existing_spend_obj["spend"] = new_spend existing_spend_obj["spend"] = new_spend
user_api_key_cache.set_cache(key=_id, value=existing_spend_obj) await user_api_key_cache.async_set_cache(
key=_id, value=existing_spend_obj
)
else: else:
existing_spend_obj.spend = new_spend existing_spend_obj.spend = new_spend
user_api_key_cache.set_cache(key=_id, value=existing_spend_obj.json()) await user_api_key_cache.async_set_cache(
key=_id, value=existing_spend_obj.json()
)
except Exception as e: except Exception as e:
verbose_proxy_logger.exception( verbose_proxy_logger.exception(
f"An error occurred updating end user cache: {str(e)}" f"An error occurred updating end user cache: {str(e)}"
@ -1279,10 +1284,14 @@ async def update_cache(
# Update the cost column for the given user # Update the cost column for the given user
if isinstance(existing_spend_obj, dict): if isinstance(existing_spend_obj, dict):
existing_spend_obj["spend"] = new_spend existing_spend_obj["spend"] = new_spend
user_api_key_cache.set_cache(key=_id, value=existing_spend_obj) await user_api_key_cache.async_set_cache(
key=_id, value=existing_spend_obj
)
else: else:
existing_spend_obj.spend = new_spend existing_spend_obj.spend = new_spend
user_api_key_cache.set_cache(key=_id, value=existing_spend_obj) await user_api_key_cache.async_set_cache(
key=_id, value=existing_spend_obj
)
except Exception as e: except Exception as e:
verbose_proxy_logger.exception( verbose_proxy_logger.exception(
f"An error occurred updating end user cache: {str(e)}" f"An error occurred updating end user cache: {str(e)}"