phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) failing usage based routing test

Browse source
This commit is contained in:
ishaan-jaff 2024-03-11 12:14:13 -07:00
parent ff4f05a3f8
commit e23c68b15a
2 changed files with 7 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

8
litellm/router_strategy/lowest_tpm_rpm.py
View file

@ -200,9 +200,11 @@ class LowestTPMLoggingHandler(CustomLogger):
if item_tpm == 0:
deployment = _deployment
break
elif item_tpm + input_tokens > _deployment_tpm or (
item in rpm_dict and rpm_dict[item] + 1 > _deployment_rpm
): # if user passed in tpm / rpm in the model_list
elif item_tpm + input_tokens > _deployment_tpm:
continue
elif (rpm_dict is not None and item in rpm_dict) and (
rpm_dict[item] + 1 > _deployment_rpm
):
continue
elif item_tpm < lowest_tpm:
lowest_tpm = item_tpm

4
litellm/tests/test_router_get_deployments.py
View file

@ -429,11 +429,11 @@ def test_usage_based_routing():
mock_response="good morning",
)
# print(response)
# print("response", response)
selection_counts[response["model"]] += 1
print(selection_counts)
# print("selection counts", selection_counts)
total_requests = sum(selection_counts.values())