diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f98862314..3d8a5a349 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -227,6 +227,14 @@ class ConfigGeneralSettings(LiteLLMBase): health_check_interval: int = Field( 300, description="background health check interval in seconds" ) + alerting: Optional[List] = Field( + None, + description="List of alerting integrations. Today, just slack - `alerting: ['slack']`", + ) + alerting_threshold: Optional[int] = Field( + None, + description="sends alerts if requests hang for 5min+", + ) class ConfigYAML(LiteLLMBase): @@ -234,6 +242,10 @@ class ConfigYAML(LiteLLMBase): Documents all the fields supported by the config.yaml """ + environment_variables: Optional[dict] = Field( + None, + description="Object to pass in additional environment variables via POST request", + ) model_list: Optional[List[ModelParams]] = Field( None, description="List of supported models on the server, with model-specific configs", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64f799dce..bdd886fc0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -225,8 +225,12 @@ async def user_api_key_auth( if is_master_key_valid: return UserAPIKeyAuth(api_key=master_key) + if route.startswith("/config/") and not is_master_key_valid: + raise Exception(f"Only admin can modify config") + if ( (route.startswith("/key/") or route.startswith("/user/")) + or route.startswith("/model/") and not is_master_key_valid and general_settings.get("allow_user_auth", False) != True ): @@ -2185,6 +2189,78 @@ async def retrieve_server_log(request: Request): #### BASIC ENDPOINTS #### +@router.post( + "/config/update", + tags=["config.yaml"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_config(config_info: ConfigYAML): + """ + For Admin UI - allows admin to update config via UI + + Currently supports modifying General Settings + LiteLLM settings + """ + global llm_router, llm_model_list, general_settings + try: + # Load existing config + if os.path.exists(f"{user_config_file_path}"): + with open(f"{user_config_file_path}", "r") as config_file: + config = yaml.safe_load(config_file) + else: + config = {} + backup_config = copy.deepcopy(config) + print_verbose(f"Loaded config: {config}") + + # update the general settings + if config_info.general_settings is not None: + config.setdefault("general_settings", {}) + updated_general_settings = config_info.general_settings.dict( + exclude_none=True + ) + config["general_settings"] = { + **updated_general_settings, + **config["general_settings"], + } + + if config_info.environment_variables is not None: + config.setdefault("environment_variables", {}) + updated_environment_variables = config_info.environment_variables + config["environment_variables"] = { + **updated_environment_variables, + **config["environment_variables"], + } + + # update the litellm settings + if config_info.litellm_settings is not None: + config.setdefault("litellm_settings", {}) + updated_litellm_settings = config_info.litellm_settings + config["litellm_settings"] = { + **updated_litellm_settings, + **config["litellm_settings"], + } + + # Save the updated config + with open(f"{user_config_file_path}", "w") as config_file: + yaml.dump(config, config_file, default_flow_style=False) + + # update Router + try: + llm_router, llm_model_list, general_settings = load_router_config( + router=llm_router, config_file_path=user_config_file_path + ) + except Exception as e: + # Rever to old config instead + with open(f"{user_config_file_path}", "w") as config_file: + yaml.dump(backup_config, config_file, default_flow_style=False) + raise HTTPException( + status_code=400, detail=f"Invalid config passed in. Errror - {str(e)}" + ) + return {"message": "Config updated successfully"} + except HTTPException as e: + raise e + except Exception as e: + traceback.print_exc() + raise HTTPException(status_code=500, detail=f"An error occurred - {str(e)}") @router.get("/config/yaml", tags=["config.yaml"]) diff --git a/ui/admin.py b/ui/admin.py index f85512f0b..d48e5edd8 100644 --- a/ui/admin.py +++ b/ui/admin.py @@ -66,90 +66,309 @@ def proxy_setup(): st.write(f"Current User Auth URL: {st.session_state['user_auth_url']}") +def add_new_model(): + import streamlit as st + import json, requests, uuid + + if ( + st.session_state.get("api_url", None) is None + and st.session_state.get("proxy_key", None) is None + ): + st.warning( + "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + ) + + model_name = st.text_input( + "Model Name - user-facing model name", placeholder="gpt-3.5-turbo" + ) + st.subheader("LiteLLM Params") + litellm_model_name = st.text_input( + "Model", placeholder="azure/gpt-35-turbo-us-east" + ) + litellm_api_key = st.text_input("API Key") + litellm_api_base = st.text_input( + "API Base", + placeholder="https://my-endpoint.openai.azure.com", + ) + litellm_api_version = st.text_input("API Version", placeholder="2023-07-01-preview") + litellm_params = json.loads( + st.text_area( + "Additional Litellm Params (JSON dictionary). [See all possible inputs](https://github.com/BerriAI/litellm/blob/3f15d7230fe8e7492c95a752963e7fbdcaf7bf98/litellm/main.py#L293)", + value={}, + ) + ) + st.subheader("Model Info") + mode_options = ("completion", "embedding", "image generation") + mode_selected = st.selectbox("Mode", mode_options) + model_info = json.loads( + st.text_area( + "Additional Model Info (JSON dictionary)", + value={}, + ) + ) + + if st.button("Submit"): + try: + model_info = { + "model_name": model_name, + "litellm_params": { + "model": litellm_model_name, + "api_key": litellm_api_key, + "api_base": litellm_api_base, + "api_version": litellm_api_version, + }, + "model_info": { + "id": str(uuid.uuid4()), + "mode": mode_selected, + }, + } + # Make the POST request to the specified URL + complete_url = "" + if st.session_state["api_url"].endswith("/"): + complete_url = f"{st.session_state['api_url']}model/new" + else: + complete_url = f"{st.session_state['api_url']}/model/new" + + headers = {"Authorization": f"Bearer {st.session_state['proxy_key']}"} + response = requests.post(complete_url, json=model_info, headers=headers) + + if response.status_code == 200: + st.success("Model added successfully!") + else: + st.error(f"Failed to add model. Status code: {response.status_code}") + + st.success("Form submitted successfully!") + except Exception as e: + raise e + + +def list_models(): + import streamlit as st + import requests + + # Check if the necessary configuration is available + if ( + st.session_state.get("api_url", None) is not None + and st.session_state.get("proxy_key", None) is not None + ): + # Make the GET request + try: + complete_url = "" + if isinstance(st.session_state["api_url"], str) and st.session_state[ + "api_url" + ].endswith("/"): + complete_url = f"{st.session_state['api_url']}models" + else: + complete_url = f"{st.session_state['api_url']}/models" + response = requests.get( + complete_url, + headers={"Authorization": f"Bearer {st.session_state['proxy_key']}"}, + ) + # Check if the request was successful + if response.status_code == 200: + models = response.json() + st.write(models) # or st.json(models) to pretty print the JSON + else: + st.error(f"Failed to get models. Status code: {response.status_code}") + except Exception as e: + st.error(f"An error occurred while requesting models: {e}") + else: + st.warning( + "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + ) + + +def create_key(): + import streamlit as st + import json, requests, uuid + + if ( + st.session_state.get("api_url", None) is None + and st.session_state.get("proxy_key", None) is None + ): + st.warning( + "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + ) + + duration = st.text_input("Duration - Can be in (h,m,s)", placeholder="1h") + + models = st.text_input("Models it can access (separated by comma)", value="") + models = models.split(",") if models else [] + + additional_params = json.loads( + st.text_area( + "Additional Key Params (JSON dictionary). [See all possible inputs](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)", + value={}, + ) + ) + + if st.button("Submit"): + try: + key_post_body = { + "duration": duration, + "models": models, + **additional_params, + } + # Make the POST request to the specified URL + complete_url = "" + if st.session_state["api_url"].endswith("/"): + complete_url = f"{st.session_state['api_url']}key/generate" + else: + complete_url = f"{st.session_state['api_url']}/key/generate" + + headers = {"Authorization": f"Bearer {st.session_state['proxy_key']}"} + response = requests.post(complete_url, json=key_post_body, headers=headers) + + if response.status_code == 200: + st.success(f"Key added successfully! - {response.json()}") + else: + st.error(f"Failed to add Key. Status code: {response.status_code}") + + st.success("Form submitted successfully!") + except Exception as e: + raise e + + +def update_config(): + if ( + st.session_state.get("api_url", None) is None + and st.session_state.get("proxy_key", None) is None + ): + st.warning( + "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + ) + + st.markdown("#### Alerting") + input_slack_webhook = st.text_input( + "Slack Webhook URL (Optional)", + value=st.session_state.get("slack_webhook", ""), + placeholder="https://hooks.slack.com/services/...", + ) + st.markdown( + "More information on Slack alerting configuration can be found in the [documentation]" + "(https://docs.litellm.ai/docs/proxy/alerting)." + ) + alerting_threshold = st.text_input( + "Alerting threshold (in seconds) (Optional)", + value=st.session_state.get("alerting_threshold", 300), + placeholder=300, + ) + st.markdown("How long to wait before a request is considered hanging") + st.markdown("#### Logging") + + enable_langfuse_logging = st.checkbox("Enable Langfuse Logging") + if enable_langfuse_logging == True: + langfuse_host_url = st.text_input( + "Langfuse Host", + value=st.session_state.get("langfuse_host", "https://cloud.langfuse.com"), + placeholder="https://cloud.langfuse.com", + ) + langfuse_public_key = st.text_input( + "Langfuse Public Key", + value=st.session_state.get("langfuse_public_key", ""), + placeholder="pk-lf-...", + ) + langfuse_secret_key = st.text_input( + "Langfuse Secret Key", + value=st.session_state.get("langfuse_secret_key", ""), + placeholder="sk-lf-...", + ) + # When the "Save" button is clicked, update the session state + if st.button("Save"): + try: + config_post_body = {} + if ( + enable_langfuse_logging == True + and langfuse_host_url is not None + and langfuse_public_key is not None + and langfuse_secret_key is not None + ): + config_post_body["litellm_settings"] = { + "success_callback": ["langfuse"] + } + config_post_body["environment_variables"] = { + "LANGFUSE_HOST": langfuse_host_url, + "LANGFUSE_PUBLIC_KEY": langfuse_public_key, + "LANGFUSE_SECRET_KEY": langfuse_secret_key, + } + if input_slack_webhook is not None and alerting_threshold is not None: + config_post_body["general_settings"] = { + "alerting": ["slack"], + "alerting_threshold": alerting_threshold, + } + config_post_body["environment_variables"] = { + "SLACK_WEBHOOK_URL": input_slack_webhook + } + + # Make the POST request to the specified URL + complete_url = "" + if st.session_state["api_url"].endswith("/"): + complete_url = f"{st.session_state.get('api_url')}config/update" + else: + complete_url = f"{st.session_state.get('api_url')}/config/update" + + headers = {"Authorization": f"Bearer {st.session_state['proxy_key']}"} + response = requests.post( + complete_url, json=config_post_body, headers=headers + ) + + if response.status_code == 200: + st.success(f"Config updated successfully! - {response.json()}") + else: + st.error( + f"Failed to update config. Status code: {response.status_code}. Error message: {response.json()['detail']}" + ) + + st.success("Form submitted successfully!") + except Exception as e: + raise e + + def admin_page(is_admin="NOT_GIVEN"): # Display the form for the admin to set the proxy URL and allowed email subdomain st.header("Admin Configuration") st.session_state.setdefault("is_admin", is_admin) # Add a navigation sidebar st.sidebar.title("Navigation") - page = st.sidebar.radio("Go to", ("Proxy Setup", "Add Models")) + page = st.sidebar.radio( + "Go to", + ( + "Connect to Proxy", + "Update Config", + "Add Models", + "List Models", + "Create Key", + "End-User Auth", + ), + ) # Display different pages based on navigation selection - if page == "Proxy Setup": + if page == "Connect to Proxy": + # Use text inputs with intermediary variables + input_api_url = st.text_input( + "Proxy Endpoint", + value=st.session_state.get("api_url", ""), + placeholder="http://0.0.0.0:8000", + ) + input_proxy_key = st.text_input( + "Proxy Key", + value=st.session_state.get("proxy_key", ""), + placeholder="sk-...", + ) + # When the "Save" button is clicked, update the session state + if st.button("Save"): + st.session_state["api_url"] = input_api_url + st.session_state["proxy_key"] = input_proxy_key + st.success("Configuration saved!") + elif page == "Update Config": + update_config() + elif page == "End-User Auth": proxy_setup() elif page == "Add Models": - if st.session_state["is_admin"] != True: - st.write("Complete Proxy Setup to add new models") - else: - proxy_key = st.text_input("Proxy Key", placeholder="sk-...") - model_name = st.text_input( - "Model Name - user-facing model name", placeholder="gpt-3.5-turbo" - ) - st.subheader("LiteLLM Params") - litellm_model_name = st.text_input( - "Model", placeholder="azure/gpt-35-turbo-us-east" - ) - litellm_api_key = st.text_input("API Key") - litellm_api_base = st.text_input( - "API Base", - placeholder="https://my-endpoint.openai.azure.com", - ) - litellm_api_version = st.text_input( - "API Version", placeholder="2023-07-01-preview" - ) - litellm_params = json.loads( - st.text_area( - "Additional Litellm Params (JSON dictionary). [See all possible inputs](https://github.com/BerriAI/litellm/blob/3f15d7230fe8e7492c95a752963e7fbdcaf7bf98/litellm/main.py#L293)", - value={}, - ) - ) - st.subheader("Model Info") - mode_options = ("completion", "embedding", "image generation") - mode_selected = st.selectbox("Mode", mode_options) - model_info = json.loads( - st.text_area( - "Additional Model Info (JSON dictionary)", - value={}, - ) - ) - - if st.button("Submit"): - try: - model_info = { - "model_name": model_name, - "litellm_params": { - "model": litellm_model_name, - "api_key": litellm_api_key, - "api_base": litellm_api_base, - "api_version": litellm_api_version, - }, - "model_info": { - "id": str(uuid.uuid4()), - "mode": mode_selected, - }, - } - print(f"model_info: {model_info}") - # Make the POST request to the specified URL - complete_url = "" - if st.session_state["proxy_url"].endswith("/"): - complete_url = f"{st.session_state['proxy_url']}model/new" - else: - complete_url = f"{st.session_state['proxy_url']}/model/new" - - headers = {"Authorization": f"Bearer {proxy_key}"} - response = requests.post( - complete_url, json=model_info, headers=headers - ) - - if response.status_code == 200: - st.success("Model added successfully!") - else: - st.error( - f"Failed to add model. Status code: {response.status_code}; Error Message: {response.json()['detail']}" - ) - - st.success("Form submitted successfully!") - except Exception as e: - raise e + add_new_model() + elif page == "List Models": + list_models() + elif page == "Create Key": + create_key() admin_page()