feat(proxy_server.py): allow admins to update config via /config/update endpoint

litellm/proxy/_types.py

@@ -227,6 +227,14 @@ class ConfigGeneralSettings(LiteLLMBase):
     health_check_interval: int = Field(
         300, description="background health check interval in seconds"
     )
+    alerting: Optional[List] = Field(
+        None,
+        description="List of alerting integrations. Today, just slack - `alerting: ['slack']`",
+    )
+    alerting_threshold: Optional[int] = Field(
+        None,
+        description="sends alerts if requests hang for 5min+",
+    )
 
 
 class ConfigYAML(LiteLLMBase):
@@ -234,6 +242,10 @@ class ConfigYAML(LiteLLMBase):
     Documents all the fields supported by the config.yaml
     """
 
+    environment_variables: Optional[dict] = Field(
+        None,
+        description="Object to pass in additional environment variables via POST request",
+    )
     model_list: Optional[List[ModelParams]] = Field(
         None,
         description="List of supported models on the server, with model-specific configs",

litellm/proxy/proxy_server.py

@@ -225,8 +225,12 @@ async def user_api_key_auth(
         if is_master_key_valid:
             return UserAPIKeyAuth(api_key=master_key)
 
+        if route.startswith("/config/") and not is_master_key_valid:
+            raise Exception(f"Only admin can modify config")
+
         if (
             (route.startswith("/key/") or route.startswith("/user/"))
+            or route.startswith("/model/")
             and not is_master_key_valid
             and general_settings.get("allow_user_auth", False) != True
         ):
@@ -2185,6 +2189,78 @@ async def retrieve_server_log(request: Request):
 
 
 #### BASIC ENDPOINTS ####
+@router.post(
+    "/config/update",
+    tags=["config.yaml"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def update_config(config_info: ConfigYAML):
+    """
+    For Admin UI - allows admin to update config via UI
+
+    Currently supports modifying General Settings + LiteLLM settings
+    """
+    global llm_router, llm_model_list, general_settings
+    try:
+        # Load existing config
+        if os.path.exists(f"{user_config_file_path}"):
+            with open(f"{user_config_file_path}", "r") as config_file:
+                config = yaml.safe_load(config_file)
+        else:
+            config = {}
+        backup_config = copy.deepcopy(config)
+        print_verbose(f"Loaded config: {config}")
+
+        # update the general settings
+        if config_info.general_settings is not None:
+            config.setdefault("general_settings", {})
+            updated_general_settings = config_info.general_settings.dict(
+                exclude_none=True
+            )
+            config["general_settings"] = {
+                **updated_general_settings,
+                **config["general_settings"],
+            }
+
+        if config_info.environment_variables is not None:
+            config.setdefault("environment_variables", {})
+            updated_environment_variables = config_info.environment_variables
+            config["environment_variables"] = {
+                **updated_environment_variables,
+                **config["environment_variables"],
+            }
+
+        # update the litellm settings
+        if config_info.litellm_settings is not None:
+            config.setdefault("litellm_settings", {})
+            updated_litellm_settings = config_info.litellm_settings
+            config["litellm_settings"] = {
+                **updated_litellm_settings,
+                **config["litellm_settings"],
+            }
+
+        # Save the updated config
+        with open(f"{user_config_file_path}", "w") as config_file:
+            yaml.dump(config, config_file, default_flow_style=False)
+
+        # update Router
+        try:
+            llm_router, llm_model_list, general_settings = load_router_config(
+                router=llm_router, config_file_path=user_config_file_path
+            )
+        except Exception as e:
+            # Rever to old config instead
+            with open(f"{user_config_file_path}", "w") as config_file:
+                yaml.dump(backup_config, config_file, default_flow_style=False)
+            raise HTTPException(
+                status_code=400, detail=f"Invalid config passed in. Errror - {str(e)}"
+            )
+        return {"message": "Config updated successfully"}
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=f"An error occurred - {str(e)}")
 
 
 @router.get("/config/yaml", tags=["config.yaml"])

ui/admin.py

@@ -66,21 +66,18 @@ def proxy_setup():
     st.write(f"Current User Auth URL: {st.session_state['user_auth_url']}")
 
 
-def admin_page(is_admin="NOT_GIVEN"):
+def add_new_model():
-    # Display the form for the admin to set the proxy URL and allowed email subdomain
+    import streamlit as st
-    st.header("Admin Configuration")
+    import json, requests, uuid
-    st.session_state.setdefault("is_admin", is_admin)
+
-    # Add a navigation sidebar
+    if (
-    st.sidebar.title("Navigation")
+        st.session_state.get("api_url", None) is None
-    page = st.sidebar.radio("Go to", ("Proxy Setup", "Add Models"))
+        and st.session_state.get("proxy_key", None) is None
-    # Display different pages based on navigation selection
+    ):
-    if page == "Proxy Setup":
+        st.warning(
-        proxy_setup()
+            "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page."
-    elif page == "Add Models":
+        )
-        if st.session_state["is_admin"] != True:
+
-            st.write("Complete Proxy Setup to add new models")
-        else:
-            proxy_key = st.text_input("Proxy Key", placeholder="sk-...")
     model_name = st.text_input(
         "Model Name - user-facing model name", placeholder="gpt-3.5-turbo"
     )
@@ -93,9 +90,7 @@ def admin_page(is_admin="NOT_GIVEN"):
         "API Base",
         placeholder="https://my-endpoint.openai.azure.com",
     )
-            litellm_api_version = st.text_input(
+    litellm_api_version = st.text_input("API Version", placeholder="2023-07-01-preview")
-                "API Version", placeholder="2023-07-01-preview"
-            )
     litellm_params = json.loads(
         st.text_area(
             "Additional Litellm Params (JSON dictionary). [See all possible inputs](https://github.com/BerriAI/litellm/blob/3f15d7230fe8e7492c95a752963e7fbdcaf7bf98/litellm/main.py#L293)",
@@ -127,24 +122,201 @@ def admin_page(is_admin="NOT_GIVEN"):
                     "mode": mode_selected,
                 },
             }
-                    print(f"model_info: {model_info}")
             # Make the POST request to the specified URL
             complete_url = ""
-                    if st.session_state["proxy_url"].endswith("/"):
+            if st.session_state["api_url"].endswith("/"):
-                        complete_url = f"{st.session_state['proxy_url']}model/new"
+                complete_url = f"{st.session_state['api_url']}model/new"
             else:
-                        complete_url = f"{st.session_state['proxy_url']}/model/new"
+                complete_url = f"{st.session_state['api_url']}/model/new"
 
-                    headers = {"Authorization": f"Bearer {proxy_key}"}
+            headers = {"Authorization": f"Bearer {st.session_state['proxy_key']}"}
-                    response = requests.post(
+            response = requests.post(complete_url, json=model_info, headers=headers)
-                        complete_url, json=model_info, headers=headers
-                    )
 
             if response.status_code == 200:
                 st.success("Model added successfully!")
+            else:
+                st.error(f"Failed to add model. Status code: {response.status_code}")
+
+            st.success("Form submitted successfully!")
+        except Exception as e:
+            raise e
+
+
+def list_models():
+    import streamlit as st
+    import requests
+
+    # Check if the necessary configuration is available
+    if (
+        st.session_state.get("api_url", None) is not None
+        and st.session_state.get("proxy_key", None) is not None
+    ):
+        # Make the GET request
+        try:
+            complete_url = ""
+            if isinstance(st.session_state["api_url"], str) and st.session_state[
+                "api_url"
+            ].endswith("/"):
+                complete_url = f"{st.session_state['api_url']}models"
+            else:
+                complete_url = f"{st.session_state['api_url']}/models"
+            response = requests.get(
+                complete_url,
+                headers={"Authorization": f"Bearer {st.session_state['proxy_key']}"},
+            )
+            # Check if the request was successful
+            if response.status_code == 200:
+                models = response.json()
+                st.write(models)  # or st.json(models) to pretty print the JSON
+            else:
+                st.error(f"Failed to get models. Status code: {response.status_code}")
+        except Exception as e:
+            st.error(f"An error occurred while requesting models: {e}")
+    else:
+        st.warning(
+            "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page."
+        )
+
+
+def create_key():
+    import streamlit as st
+    import json, requests, uuid
+
+    if (
+        st.session_state.get("api_url", None) is None
+        and st.session_state.get("proxy_key", None) is None
+    ):
+        st.warning(
+            "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page."
+        )
+
+    duration = st.text_input("Duration - Can be in (h,m,s)", placeholder="1h")
+
+    models = st.text_input("Models it can access (separated by comma)", value="")
+    models = models.split(",") if models else []
+
+    additional_params = json.loads(
+        st.text_area(
+            "Additional Key Params (JSON dictionary). [See all possible inputs](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)",
+            value={},
+        )
+    )
+
+    if st.button("Submit"):
+        try:
+            key_post_body = {
+                "duration": duration,
+                "models": models,
+                **additional_params,
+            }
+            # Make the POST request to the specified URL
+            complete_url = ""
+            if st.session_state["api_url"].endswith("/"):
+                complete_url = f"{st.session_state['api_url']}key/generate"
+            else:
+                complete_url = f"{st.session_state['api_url']}/key/generate"
+
+            headers = {"Authorization": f"Bearer {st.session_state['proxy_key']}"}
+            response = requests.post(complete_url, json=key_post_body, headers=headers)
+
+            if response.status_code == 200:
+                st.success(f"Key added successfully! - {response.json()}")
+            else:
+                st.error(f"Failed to add Key. Status code: {response.status_code}")
+
+            st.success("Form submitted successfully!")
+        except Exception as e:
+            raise e
+
+
+def update_config():
+    if (
+        st.session_state.get("api_url", None) is None
+        and st.session_state.get("proxy_key", None) is None
+    ):
+        st.warning(
+            "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page."
+        )
+
+    st.markdown("#### Alerting")
+    input_slack_webhook = st.text_input(
+        "Slack Webhook URL (Optional)",
+        value=st.session_state.get("slack_webhook", ""),
+        placeholder="https://hooks.slack.com/services/...",
+    )
+    st.markdown(
+        "More information on Slack alerting configuration can be found in the [documentation]"
+        "(https://docs.litellm.ai/docs/proxy/alerting)."
+    )
+    alerting_threshold = st.text_input(
+        "Alerting threshold (in seconds) (Optional)",
+        value=st.session_state.get("alerting_threshold", 300),
+        placeholder=300,
+    )
+    st.markdown("How long to wait before a request is considered hanging")
+    st.markdown("#### Logging")
+
+    enable_langfuse_logging = st.checkbox("Enable Langfuse Logging")
+    if enable_langfuse_logging == True:
+        langfuse_host_url = st.text_input(
+            "Langfuse Host",
+            value=st.session_state.get("langfuse_host", "https://cloud.langfuse.com"),
+            placeholder="https://cloud.langfuse.com",
+        )
+        langfuse_public_key = st.text_input(
+            "Langfuse Public Key",
+            value=st.session_state.get("langfuse_public_key", ""),
+            placeholder="pk-lf-...",
+        )
+        langfuse_secret_key = st.text_input(
+            "Langfuse Secret Key",
+            value=st.session_state.get("langfuse_secret_key", ""),
+            placeholder="sk-lf-...",
+        )
+    # When the "Save" button is clicked, update the session state
+    if st.button("Save"):
+        try:
+            config_post_body = {}
+            if (
+                enable_langfuse_logging == True
+                and langfuse_host_url is not None
+                and langfuse_public_key is not None
+                and langfuse_secret_key is not None
+            ):
+                config_post_body["litellm_settings"] = {
+                    "success_callback": ["langfuse"]
+                }
+                config_post_body["environment_variables"] = {
+                    "LANGFUSE_HOST": langfuse_host_url,
+                    "LANGFUSE_PUBLIC_KEY": langfuse_public_key,
+                    "LANGFUSE_SECRET_KEY": langfuse_secret_key,
+                }
+            if input_slack_webhook is not None and alerting_threshold is not None:
+                config_post_body["general_settings"] = {
+                    "alerting": ["slack"],
+                    "alerting_threshold": alerting_threshold,
+                }
+                config_post_body["environment_variables"] = {
+                    "SLACK_WEBHOOK_URL": input_slack_webhook
+                }
+
+            # Make the POST request to the specified URL
+            complete_url = ""
+            if st.session_state["api_url"].endswith("/"):
+                complete_url = f"{st.session_state.get('api_url')}config/update"
+            else:
+                complete_url = f"{st.session_state.get('api_url')}/config/update"
+
+            headers = {"Authorization": f"Bearer {st.session_state['proxy_key']}"}
+            response = requests.post(
+                complete_url, json=config_post_body, headers=headers
+            )
+
+            if response.status_code == 200:
+                st.success(f"Config updated successfully! - {response.json()}")
             else:
                 st.error(
-                            f"Failed to add model. Status code: {response.status_code}; Error Message: {response.json()['detail']}"
+                    f"Failed to update config. Status code: {response.status_code}. Error message: {response.json()['detail']}"
                 )
 
             st.success("Form submitted successfully!")
@@ -152,4 +324,51 @@ def admin_page(is_admin="NOT_GIVEN"):
             raise e
 
 
+def admin_page(is_admin="NOT_GIVEN"):
+    # Display the form for the admin to set the proxy URL and allowed email subdomain
+    st.header("Admin Configuration")
+    st.session_state.setdefault("is_admin", is_admin)
+    # Add a navigation sidebar
+    st.sidebar.title("Navigation")
+    page = st.sidebar.radio(
+        "Go to",
+        (
+            "Connect to Proxy",
+            "Update Config",
+            "Add Models",
+            "List Models",
+            "Create Key",
+            "End-User Auth",
+        ),
+    )
+    # Display different pages based on navigation selection
+    if page == "Connect to Proxy":
+        # Use text inputs with intermediary variables
+        input_api_url = st.text_input(
+            "Proxy Endpoint",
+            value=st.session_state.get("api_url", ""),
+            placeholder="http://0.0.0.0:8000",
+        )
+        input_proxy_key = st.text_input(
+            "Proxy Key",
+            value=st.session_state.get("proxy_key", ""),
+            placeholder="sk-...",
+        )
+        # When the "Save" button is clicked, update the session state
+        if st.button("Save"):
+            st.session_state["api_url"] = input_api_url
+            st.session_state["proxy_key"] = input_proxy_key
+            st.success("Configuration saved!")
+    elif page == "Update Config":
+        update_config()
+    elif page == "End-User Auth":
+        proxy_setup()
+    elif page == "Add Models":
+        add_new_model()
+    elif page == "List Models":
+        list_models()
+    elif page == "Create Key":
+        create_key()
+
+
 admin_page()