build(litellm_server): add support for global settings

litellm_server/main.py

@@ -39,6 +39,8 @@ else:
 @router.get("/models")  # if project requires model list
 def model_list():
     all_models = litellm.utils.get_valid_models()
+    if llm_model_list: 
+        all_models += llm_model_list
     return dict(
         data=[
             {

litellm_server/utils.py

@@ -55,6 +55,12 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: Optio
     except:
         pass
 
+    ## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
+    litellm_settings = config.get('litellm_settings', None)
+    if litellm_settings: 
+        for key, value in litellm_settings.items(): 
+            setattr(litellm, key, value)
+
     ## MODEL LIST
     model_list = config.get('model_list', None)
     if model_list: 

router_config_template.yaml

@@ -1,26 +1,28 @@
+# Global settings for the litellm module
+litellm_settings:
+  drop_params: True
+  # failure_callbacks: ["sentry"]
+
+# Model-specific settings
 model_list: # refer to https://docs.litellm.ai/docs/routing
   - model_name: gpt-3.5-turbo
-    litellm_params:
+    litellm_params:  # parameters for litellm.completion() 
       model: azure/chatgpt-v-2 # azure/<your-deployment-name>
       api_key: your_azure_api_key
       api_version: your_azure_api_version
       api_base: your_azure_api_base
-    tpm: 240000 # REPLACE with your azure deployment tpm
-    rpm: 1800 # REPLACE with your azure deployment rpm
-  - model_name: gpt-3.5-turbo
+    tpm: 240000 # [OPTIONAL] To load balance between multiple deployments
+    rpm: 1800 # [OPTIONAL] To load balance between multiple deployments
+  - model_name: mistral
     litellm_params:
-      model: azure/chatgpt-functioncalling
-      api_key: your_azure_api_key
-      api_version: your_azure_api_version
-      api_base: your_azure_api_base
-    tpm: 240000
-    rpm: 1800
+      model: ollama/mistral
+      api_base: my_ollama_api_base
   - model_name: gpt-3.5-turbo
     litellm_params:
       model: gpt-3.5-turbo
       api_key: your_openai_api_key
-    tpm: 1000000 # REPLACE with your openai tpm
-    rpm: 9000 # REPLACE with your openai rpm
+    tpm: 1000000 # [OPTIONAL] REPLACE with your openai tpm
+    rpm: 9000 # [OPTIONAL] REPLACE with your openai rpm
 
 environment_variables:
   REDIS_HOST: your_redis_host