phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

docs(routing.md): adding router to docs

Browse source
This commit is contained in:
Krrish Dholakia 2023-10-18 14:20:45 -07:00
parent a0f6279bff
commit bace883dcf
2 changed files with 51 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

50
docs/my-website/docs/routing.md Normal file
View file

@ -0,0 +1,50 @@
# Load-Balance Router
Use this if you're trying to load-balance across multiple deployments.
```python
pip install litellm
```
```python
from litellm import Router
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE")
},
"tpm": 240000,
"rpm": 1800
}, {
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE")
},
"tpm": 240000,
"rpm": 1800
}, {
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "gpt-3.5-turbo",
"api_key": os.getenv("OPENAI_API_KEY"),
},
"tpm": 1000000,
"rpm": 9000
}]
router = Router(model_list=model_list)
# openai.ChatCompletion.create replacement
response = router.completion(model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
print(response)
```

2
docs/my-website/sidebars.js
View file

@ -83,7 +83,7 @@ const sidebars = {
},
"proxy_server",
"budget_manager",
"rate_limit_manager",
"routing",
"set_keys",
"completion/token_usage",
{