phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) router: set default rpm/tpm when not set

Browse source
This commit is contained in:
ishaan-jaff 2023-11-29 18:13:27 -08:00
parent c1914a01bc
commit 7bcc23e8e9
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/router.py
View file

@ -941,7 +941,7 @@ class Router:
rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
if rpm is not None:
# use weight-random pick if rpms provided
rpms = [m["litellm_params"].get("rpm") for m in healthy_deployments]
rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
self.print_verbose(f"\nrpms {rpms}")
total_rpm = sum(rpms)
weights = [rpm / total_rpm for rpm in rpms]
@ -955,7 +955,7 @@ class Router:
tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
if tpm is not None:
# use weight-random pick if rpms provided
tpms = [m["litellm_params"].get("tpm") for m in healthy_deployments]
tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
self.print_verbose(f"\ntpms {tpms}")
total_tpm = sum(tpms)
weights = [tpm / total_tpm for tpm in tpms]