fix(router.py): fix client init for streaming timeouts

This commit is contained in:
Krrish Dholakia 2024-04-05 12:06:46 -07:00 committed by Ishaan Jaff
parent f0c4ff6e60
commit 695eb129ad
4 changed files with 29 additions and 7 deletions

View file

@ -9,12 +9,18 @@ model_list:
litellm_params:
model: "gpt-3.5-turbo-1106"
api_key: os.environ/OPENAI_API_KEY
rpm: 480
timeout: 300
stream_timeout: 60
- model_name: gpt-4
litellm_params:
model: azure/chatgpt-v-2
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
api_version: "2023-05-15"
api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
rpm: 480
timeout: 300
stream_timeout: 60
- model_name: sagemaker-completion-model
litellm_params:
model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4