forked from phoenix/litellm-mirror
docs(routing.md): adding router to docs
This commit is contained in:
parent
a0f6279bff
commit
bace883dcf
2 changed files with 51 additions and 1 deletions
50
docs/my-website/docs/routing.md
Normal file
50
docs/my-website/docs/routing.md
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
# Load-Balance Router
|
||||||
|
|
||||||
|
Use this if you're trying to load-balance across multiple deployments.
|
||||||
|
|
||||||
|
|
||||||
|
```python
|
||||||
|
pip install litellm
|
||||||
|
```
|
||||||
|
|
||||||
|
```python
|
||||||
|
from litellm import Router
|
||||||
|
|
||||||
|
model_list = [{ # list of model deployments
|
||||||
|
"model_name": "gpt-3.5-turbo", # openai model name
|
||||||
|
"litellm_params": { # params for litellm completion/embedding call
|
||||||
|
"model": "azure/chatgpt-v-2",
|
||||||
|
"api_key": os.getenv("AZURE_API_KEY"),
|
||||||
|
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||||
|
"api_base": os.getenv("AZURE_API_BASE")
|
||||||
|
},
|
||||||
|
"tpm": 240000,
|
||||||
|
"rpm": 1800
|
||||||
|
}, {
|
||||||
|
"model_name": "gpt-3.5-turbo", # openai model name
|
||||||
|
"litellm_params": { # params for litellm completion/embedding call
|
||||||
|
"model": "azure/chatgpt-functioncalling",
|
||||||
|
"api_key": os.getenv("AZURE_API_KEY"),
|
||||||
|
"api_version": os.getenv("AZURE_API_VERSION"),
|
||||||
|
"api_base": os.getenv("AZURE_API_BASE")
|
||||||
|
},
|
||||||
|
"tpm": 240000,
|
||||||
|
"rpm": 1800
|
||||||
|
}, {
|
||||||
|
"model_name": "gpt-3.5-turbo", # openai model name
|
||||||
|
"litellm_params": { # params for litellm completion/embedding call
|
||||||
|
"model": "gpt-3.5-turbo",
|
||||||
|
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||||
|
},
|
||||||
|
"tpm": 1000000,
|
||||||
|
"rpm": 9000
|
||||||
|
}]
|
||||||
|
|
||||||
|
router = Router(model_list=model_list)
|
||||||
|
|
||||||
|
# openai.ChatCompletion.create replacement
|
||||||
|
response = router.completion(model="gpt-3.5-turbo",
|
||||||
|
messages=[{"role": "user", "content": "Hey, how's it going?"}]
|
||||||
|
|
||||||
|
print(response)
|
||||||
|
```
|
|
@ -83,7 +83,7 @@ const sidebars = {
|
||||||
},
|
},
|
||||||
"proxy_server",
|
"proxy_server",
|
||||||
"budget_manager",
|
"budget_manager",
|
||||||
"rate_limit_manager",
|
"routing",
|
||||||
"set_keys",
|
"set_keys",
|
||||||
"completion/token_usage",
|
"completion/token_usage",
|
||||||
{
|
{
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue