phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 10:14:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) router: set default rpm/tpm when not set

Browse source
This commit is contained in:
ishaan-jaff 2023-11-29 18:13:27 -08:00
parent c1914a01bc
commit 7bcc23e8e9
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/router.py
View file

@ -941,7 +941,7 @@ class Router:
rpm = healthy_deployments[0].get("litellm_params").get("rpm", None) rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
if rpm is not None: if rpm is not None:
# use weight-random pick if rpms provided # use weight-random pick if rpms provided
rpms = [m["litellm_params"].get("rpm") for m in healthy_deployments] rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
self.print_verbose(f"\nrpms {rpms}") self.print_verbose(f"\nrpms {rpms}")
total_rpm = sum(rpms) total_rpm = sum(rpms)
weights = [rpm / total_rpm for rpm in rpms] weights = [rpm / total_rpm for rpm in rpms]
@ -955,7 +955,7 @@ class Router:
tpm = healthy_deployments[0].get("litellm_params").get("tpm", None) tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
if tpm is not None: if tpm is not None:
# use weight-random pick if rpms provided # use weight-random pick if rpms provided
tpms = [m["litellm_params"].get("tpm") for m in healthy_deployments] tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
self.print_verbose(f"\ntpms {tpms}") self.print_verbose(f"\ntpms {tpms}")
total_tpm = sum(tpms) total_tpm = sum(tpms)
weights = [tpm / total_tpm for tpm in tpms] weights = [tpm / total_tpm for tpm in tpms]