From 0ee9c61090f6acaaea27b00aa13a441d2081774b Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Fri, 27 Oct 2023 16:24:54 -0700
Subject: [PATCH] build(litellm_server): add support for global settings

---
 litellm_server/main.py      |  2 ++
 litellm_server/utils.py     |  6 ++++++
 router_config_template.yaml | 26 ++++++++++++++------------
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/litellm_server/main.py b/litellm_server/main.py
index 8593631c3..76eab9ead 100644
--- a/litellm_server/main.py
+++ b/litellm_server/main.py
@@ -39,6 +39,8 @@ else:
 @router.get("/models")  # if project requires model list
 def model_list():
     all_models = litellm.utils.get_valid_models()
+    if llm_model_list: 
+        all_models += llm_model_list
     return dict(
         data=[
             {
diff --git a/litellm_server/utils.py b/litellm_server/utils.py
index 359f4ab5f..3fb656154 100644
--- a/litellm_server/utils.py
+++ b/litellm_server/utils.py
@@ -55,6 +55,12 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: Optio
     except:
         pass
 
+    ## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
+    litellm_settings = config.get('litellm_settings', None)
+    if litellm_settings: 
+        for key, value in litellm_settings.items(): 
+            setattr(litellm, key, value)
+
     ## MODEL LIST
     model_list = config.get('model_list', None)
     if model_list: 
diff --git a/router_config_template.yaml b/router_config_template.yaml
index e548f9829..b6a8612a4 100644
--- a/router_config_template.yaml
+++ b/router_config_template.yaml
@@ -1,26 +1,28 @@
+# Global settings for the litellm module
+litellm_settings:
+  drop_params: True
+  # failure_callbacks: ["sentry"]
+
+# Model-specific settings
 model_list: # refer to https://docs.litellm.ai/docs/routing
   - model_name: gpt-3.5-turbo
-    litellm_params:
+    litellm_params:  # parameters for litellm.completion() 
       model: azure/chatgpt-v-2 # azure/<your-deployment-name>
       api_key: your_azure_api_key
       api_version: your_azure_api_version
       api_base: your_azure_api_base
-    tpm: 240000 # REPLACE with your azure deployment tpm
-    rpm: 1800 # REPLACE with your azure deployment rpm
-  - model_name: gpt-3.5-turbo
+    tpm: 240000 # [OPTIONAL] To load balance between multiple deployments
+    rpm: 1800 # [OPTIONAL] To load balance between multiple deployments
+  - model_name: mistral
     litellm_params:
-      model: azure/chatgpt-functioncalling
-      api_key: your_azure_api_key
-      api_version: your_azure_api_version
-      api_base: your_azure_api_base
-    tpm: 240000
-    rpm: 1800
+      model: ollama/mistral
+      api_base: my_ollama_api_base
   - model_name: gpt-3.5-turbo
     litellm_params:
       model: gpt-3.5-turbo
       api_key: your_openai_api_key
-    tpm: 1000000 # REPLACE with your openai tpm
-    rpm: 9000 # REPLACE with your openai rpm
+    tpm: 1000000 # [OPTIONAL] REPLACE with your openai tpm
+    rpm: 9000 # [OPTIONAL] REPLACE with your openai rpm
 
 environment_variables:
   REDIS_HOST: your_redis_host