phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

docs(routing.md): add queueing to docs

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-21 18:00:14 -08:00
parent a2681e353f
commit 9d97082eed
9 changed files with 244 additions and 261 deletions
Download patch file Download diff file Expand all files Collapse all files

11
cookbook/litellm_router/load_test_queuing.py
View file

@ -13,7 +13,7 @@ import pytest
from litellm import Router
import litellm
litellm.set_verbose=False
os.environ.pop("AZURE_AD_TOKEN")
# os.environ.pop("AZURE_AD_TOKEN")
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # model alias
@ -142,10 +142,11 @@ successful_calls = 0
failed_calls = 0
for future in futures:
if future.result() is not None:
successful_calls += 1
else:
failed_calls += 1
if future.done():
if future.result() is not None:
successful_calls += 1
else:
failed_calls += 1
print(f"Load test Summary:")
print(f"Total Requests: {concurrent_calls}")