diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ca9926cef..f1d824b86 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4,6 +4,7 @@ import enum from typing import Optional, List, Union, Dict, Literal, Any from datetime import datetime import uuid, json, sys, os +from litellm.types.router import UpdateRouterConfig def hash_token(token: str): @@ -750,7 +751,7 @@ class ConfigYAML(LiteLLMBase): description="litellm Module settings. See __init__.py for all, example litellm.drop_params=True, litellm.set_verbose=True, litellm.api_base, litellm.cache", ) general_settings: Optional[ConfigGeneralSettings] = None - router_settings: Optional[dict] = Field( + router_settings: Optional[UpdateRouterConfig] = Field( None, description="litellm router object settings. See router.py __init__ for all, example router.num_retries=5, router.timeout=5, router.max_retries=5, router.retry_after=5", ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1419a963b..d840b29c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2636,9 +2636,16 @@ class ProxyConfig: ] # router settings - if llm_router is not None: - _router_settings = config_data.get("router_settings", {}) - llm_router.update_settings(**_router_settings) + if llm_router is not None and prisma_client is not None: + db_router_settings = await prisma_client.db.litellm_config.find_first( + where={"param_name": "router_settings"} + ) + if ( + db_router_settings is not None + and db_router_settings.param_value is not None + ): + _router_settings = db_router_settings.param_value + llm_router.update_settings(**_router_settings) async def add_deployment( self, @@ -8406,96 +8413,29 @@ async def update_config(config_info: ConfigYAML): """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client try: - import base64 - # Load existing config - config = await proxy_config.get_config() - verbose_proxy_logger.debug("Loaded config: %s", config) + """ + - Update the ConfigTable DB + - Run 'add_deployment' + """ + if prisma_client is None: + raise Exception("No DB Connected") - # update the general settings - if config_info.general_settings is not None: - config.setdefault("general_settings", {}) - updated_general_settings = config_info.general_settings.dict( - exclude_none=True + updated_settings = config_info.json(exclude_none=True) + updated_settings = prisma_client.jsonify_object(updated_settings) + for k, v in updated_settings.items(): + await prisma_client.db.litellm_config.upsert( + where={"param_name": k}, + data={ + "create": {"param_name": k, "param_value": v}, + "update": {"param_value": v}, + }, ) - _existing_settings = config["general_settings"] - for k, v in updated_general_settings.items(): - # overwrite existing settings with updated values - _existing_settings[k] = v - config["general_settings"] = _existing_settings + await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) - if config_info.environment_variables is not None: - config.setdefault("environment_variables", {}) - _updated_environment_variables = config_info.environment_variables - - # encrypt updated_environment_variables # - for k, v in _updated_environment_variables.items(): - if isinstance(v, str): - encrypted_value = encrypt_value(value=v, master_key=master_key) # type: ignore - _updated_environment_variables[k] = base64.b64encode( - encrypted_value - ).decode("utf-8") - - _existing_env_variables = config["environment_variables"] - - for k, v in _updated_environment_variables.items(): - # overwrite existing env variables with updated values - _existing_env_variables[k] = _updated_environment_variables[k] - - # update the litellm settings - if config_info.litellm_settings is not None: - config.setdefault("litellm_settings", {}) - updated_litellm_settings = config_info.litellm_settings - config["litellm_settings"] = { - **updated_litellm_settings, - **config["litellm_settings"], - } - - # if litellm.success_callback in updated_litellm_settings and config["litellm_settings"] - if ( - "success_callback" in updated_litellm_settings - and "success_callback" in config["litellm_settings"] - ): - - # check both success callback are lists - if isinstance( - config["litellm_settings"]["success_callback"], list - ) and isinstance(updated_litellm_settings["success_callback"], list): - combined_success_callback = ( - config["litellm_settings"]["success_callback"] - + updated_litellm_settings["success_callback"] - ) - combined_success_callback = list(set(combined_success_callback)) - config["litellm_settings"][ - "success_callback" - ] = combined_success_callback - - # router settings - if config_info.router_settings is not None: - config.setdefault("router_settings", {}) - _updated_router_settings = config_info.router_settings - - config["router_settings"] = { - **config["router_settings"], - **_updated_router_settings, - } - - # Save the updated config - await proxy_config.save_config(new_config=config) - - # make sure the change is instantly rolled out for langfuse - if prisma_client is not None: - await proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj - ) - - # Test new connections - ## Slack - if "slack" in config.get("general_settings", {}).get("alerting", []): - await proxy_logging_obj.alerting_handler( - message="This is a test", level="Low" - ) return {"message": "Config updated successfully"} except Exception as e: traceback.print_exc() diff --git a/litellm/router.py b/litellm/router.py index da871daba..2a710f5cf 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2553,6 +2553,8 @@ class Router: "timeout", "max_retries", "retry_after", + "fallbacks", + "context_window_fallbacks", ] _int_settings = [ diff --git a/litellm/types/router.py b/litellm/types/router.py index 961f20a91..c5ec47091 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -48,6 +48,23 @@ class RouterConfig(BaseModel): protected_namespaces = () +class UpdateRouterConfig(BaseModel): + """ + Set of params that you can modify via `router.update_settings()`. + """ + + routing_strategy_args: Optional[dict] = None + routing_strategy: Optional[str] = None + allowed_fails: Optional[int] = None + cooldown_time: Optional[float] = None + num_retries: Optional[int] = None + timeout: Optional[float] = None + max_retries: Optional[int] = None + retry_after: Optional[float] = None + fallbacks: Optional[List[dict]] = None + context_window_fallbacks: Optional[List[dict]] = None + + class ModelInfo(BaseModel): id: Optional[ str diff --git a/tests/test_adding_callbacks.py b/tests/test_config.py similarity index 100% rename from tests/test_adding_callbacks.py rename to tests/test_config.py diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index e227568d0..ace755a2c 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -125,7 +125,7 @@ const GeneralSettings: React.FC = ({ - {Object.entries(routerSettings).map(([param, value]) => ( + {Object.entries(routerSettings).filter(([param, value]) => param != "fallbacks" && param != "context_window_fallbacks").map(([param, value]) => ( {param}