phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): fix caching for tracking cooldowns + usage

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-23 11:13:24 -08:00
parent 94c1d71b2c
commit 61fc76a8c4
5 changed files with 148 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

14
litellm/tests/test_router.py
View file

@ -18,7 +18,7 @@ load_dotenv()
def test_multiple_deployments():
import concurrent, time
litellm.set_verbose=True
litellm.set_verbose=False
futures = {}
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # openai model name
@ -58,6 +58,7 @@ def test_multiple_deployments():
redis_password=os.getenv("REDIS_PASSWORD"),
redis_port=int(os.getenv("REDIS_PORT")),
routing_strategy="simple-shuffle",
set_verbose=False,
num_retries=1) # type: ignore
# router = Router(model_list=model_list, redis_host=os.getenv("REDIS_HOST"), redis_password=os.getenv("REDIS_PASSWORD"), redis_port=int(os.getenv("REDIS_PORT"))) # type: ignore
kwargs = {
@ -81,12 +82,13 @@ Who among the mentioned figures from Ancient Greece contributed to the domain of
}
results = []
for _ in range(2):
print(f"starting!!!")
response = router.completion(**kwargs)
results.append(response)
try:
for _ in range(3):
response = router.completion(**kwargs)
results.append(response)
except Exception as e:
raise e
# print(len(results))
# with ThreadPoolExecutor(max_workers=100) as executor: