phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add litellm-key-remaining-tokens on prometheus

Browse source
This commit is contained in:
Ishaan Jaff 2024-08-17 10:02:20 -07:00
parent 785482f023
commit 412d30d362
3 changed files with 56 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -263,6 +263,17 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
request_count_api_key, new_val
)
tpm_limit_for_model = tpm_limit_for_model or sys.maxsize
rpm_limit_for_model = rpm_limit_for_model or sys.maxsize
# Add remaining tokens, requests to metadata
_remaining_tokens = tpm_limit_for_model - new_val["current_tpm"]
_remaining_requests = rpm_limit_for_model - new_val["current_rpm"]
_remaining_limits_data = {
f"litellm-key-remaining-tokens-{_model}": _remaining_tokens,
f"litellm-key-remaining-requests-{_model}": _remaining_requests,
}
data["metadata"].update(_remaining_limits_data)
# check if REQUEST ALLOWED for user_id
user_id = user_api_key_dict.user_id
if user_id is not None: