phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat return rmng tokens for model for api key

Browse source
This commit is contained in:
Ishaan Jaff 2024-08-17 12:35:10 -07:00
parent 5985c7e933
commit ee0f772b5c
3 changed files with 73 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -268,11 +268,14 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
request_count_api_key, new_val
)
tpm_limit_for_model = tpm_limit_for_model or sys.maxsize
rpm_limit_for_model = rpm_limit_for_model or sys.maxsize
_remaining_tokens = None
_remaining_requests = None
# Add remaining tokens, requests to metadata
_remaining_tokens = tpm_limit_for_model - new_val["current_tpm"]
_remaining_requests = rpm_limit_for_model - new_val["current_rpm"]
if tpm_limit_for_model is not None:
_remaining_tokens = tpm_limit_for_model - new_val["current_tpm"]
if rpm_limit_for_model is not None:
_remaining_requests = rpm_limit_for_model - new_val["current_rpm"]
_remaining_limits_data = {
f"litellm-key-remaining-tokens-{_model}": _remaining_tokens,
f"litellm-key-remaining-requests-{_model}": _remaining_requests,