phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 18:24:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

build(litellm_server): add support for global settings

Browse source
This commit is contained in:
Krrish Dholakia 2023-10-27 16:24:54 -07:00
parent e54f5d801e
commit 0ee9c61090
3 changed files with 22 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm_server/main.py
View file

@ -39,6 +39,8 @@ else:
@router.get("/models") # if project requires model list @router.get("/models") # if project requires model list
def model_list(): def model_list():
all_models = litellm.utils.get_valid_models() all_models = litellm.utils.get_valid_models()
if llm_model_list:
all_models += llm_model_list
return dict( return dict(
data=[ data=[
{ {

6
litellm_server/utils.py
View file

@ -55,6 +55,12 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: Optio
except: except:
pass pass
## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
litellm_settings = config.get('litellm_settings', None)
if litellm_settings:
for key, value in litellm_settings.items():
setattr(litellm, key, value)
## MODEL LIST ## MODEL LIST
model_list = config.get('model_list', None) model_list = config.get('model_list', None)
if model_list: if model_list:

26
router_config_template.yaml
View file

@ -1,26 +1,28 @@
# Global settings for the litellm module
litellm_settings:
drop_params: True
# failure_callbacks: ["sentry"]
# Model-specific settings
model_list: # refer to https://docs.litellm.ai/docs/routing model_list: # refer to https://docs.litellm.ai/docs/routing
- model_name: gpt-3.5-turbo - model_name: gpt-3.5-turbo
litellm_params: litellm_params: # parameters for litellm.completion()
model: azure/chatgpt-v-2 # azure/<your-deployment-name> model: azure/chatgpt-v-2 # azure/<your-deployment-name>
api_key: your_azure_api_key api_key: your_azure_api_key
api_version: your_azure_api_version api_version: your_azure_api_version
api_base: your_azure_api_base api_base: your_azure_api_base
tpm: 240000 # REPLACE with your azure deployment tpm tpm: 240000 # [OPTIONAL] To load balance between multiple deployments
rpm: 1800 # REPLACE with your azure deployment rpm rpm: 1800 # [OPTIONAL] To load balance between multiple deployments
- model_name: gpt-3.5-turbo - model_name: mistral
litellm_params: litellm_params:
model: azure/chatgpt-functioncalling model: ollama/mistral
api_key: your_azure_api_key api_base: my_ollama_api_base
api_version: your_azure_api_version
api_base: your_azure_api_base
tpm: 240000
rpm: 1800
- model_name: gpt-3.5-turbo - model_name: gpt-3.5-turbo
litellm_params: litellm_params:
model: gpt-3.5-turbo model: gpt-3.5-turbo
api_key: your_openai_api_key api_key: your_openai_api_key
tpm: 1000000 # REPLACE with your openai tpm tpm: 1000000 # [OPTIONAL] REPLACE with your openai tpm
rpm: 9000 # REPLACE with your openai rpm rpm: 9000 # [OPTIONAL] REPLACE with your openai rpm
environment_variables: environment_variables:
REDIS_HOST: your_redis_host REDIS_HOST: your_redis_host