Merge branch 'main' into litellm_send_alerts_making_new_key

litellm/integrations/slack_alerting.py

@@ -1,7 +1,7 @@
 #### What this does ####
 #    Class for sending Slack Alerts #
-import dotenv, os
+import dotenv, os, traceback
-from litellm.proxy._types import UserAPIKeyAuth, CallInfo
+from litellm.proxy._types import UserAPIKeyAuth, CallInfo, AlertType
 from litellm._logging import verbose_logger, verbose_proxy_logger
 import litellm, threading
 from typing import List, Literal, Any, Union, Optional, Dict
@@ -16,6 +16,20 @@ from datetime import datetime as dt, timedelta, timezone
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import WebhookEvent
 import random
+from typing import TypedDict
+from openai import APIError
+
+import litellm.types
+import litellm.types.router
+
+
+class OutageModel(TypedDict):
+    model_id: str
+    alerts: List[int]
+    deployment_ids: List[str]
+    minor_alert_sent: bool
+    major_alert_sent: bool
+    last_updated_at: float
 
 # we use this for the email header, please send a test email if you change this. verify it looks good on email
 LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
@@ -45,6 +59,10 @@ class SlackAlertingArgs(LiteLLMBase):
     )
     report_check_interval: int = 5 * 60  # 5 minutes
     budget_alert_ttl: int = 24 * 60 * 60  # 24 hours
+    outage_alert_ttl: int = 1 * 60  # 1 minute ttl
+    minor_outage_alert_threshold: int = 5
+    major_outage_alert_threshold: int = 10
+    max_outage_alert_list_size: int = 10  # prevent memory leak
 
 
 class DeploymentMetrics(LiteLLMBase):
@@ -88,19 +106,7 @@ class SlackAlerting(CustomLogger):
         internal_usage_cache: Optional[DualCache] = None,
         alerting_threshold: float = 300,  # threshold for slow / hanging llm responses (in seconds)
         alerting: Optional[List] = [],
-        alert_types: List[
+        alert_types: List[AlertType] = [
-            Literal[
-                "llm_exceptions",
-                "llm_too_slow",
-                "llm_requests_hanging",
-                "budget_alerts",
-                "db_exceptions",
-                "daily_reports",
-                "spend_reports",
-                "cooldown_deployment",
-                "new_model_added",
-            ]
-        ] = [
             "llm_exceptions",
             "llm_too_slow",
             "llm_requests_hanging",
@@ -110,6 +116,7 @@ class SlackAlerting(CustomLogger):
             "spend_reports",
             "cooldown_deployment",
             "new_model_added",
+            "outage_alerts",
         ],
         alert_to_webhook_url: Optional[
             Dict
@@ -126,6 +133,7 @@ class SlackAlerting(CustomLogger):
         self.is_running = False
         self.alerting_args = SlackAlertingArgs(**alerting_args)
         self.default_webhook_url = default_webhook_url
+        self.llm_router: Optional[litellm.Router] = None
 
     def update_values(
         self,
@@ -134,6 +142,7 @@ class SlackAlerting(CustomLogger):
         alert_types: Optional[List] = None,
         alert_to_webhook_url: Optional[Dict] = None,
         alerting_args: Optional[Dict] = None,
+        llm_router: Optional[litellm.Router] = None,
     ):
         if alerting is not None:
             self.alerting = alerting
@@ -149,6 +158,8 @@ class SlackAlerting(CustomLogger):
                 self.alert_to_webhook_url = alert_to_webhook_url
             else:
                 self.alert_to_webhook_url.update(alert_to_webhook_url)
+        if llm_router is not None:
+            self.llm_router = llm_router
 
     async def deployment_in_cooldown(self):
         pass
@@ -701,6 +712,158 @@ class SlackAlerting(CustomLogger):
             return
         return
 
+    def _count_outage_alerts(self, alerts: List[int]) -> str:
+        """
+        Parameters:
+        - alerts: List[int] -> list of error codes (either 408 or 500+)
+
+        Returns:
+        - str -> formatted string. This is an alert message, giving a human-friendly description of the errors.
+        """
+        error_breakdown = {"Timeout Errors": 0, "API Errors": 0, "Unknown Errors": 0}
+        for alert in alerts:
+            if alert == 408:
+                error_breakdown["Timeout Errors"] += 1
+            elif alert >= 500:
+                error_breakdown["API Errors"] += 1
+            else:
+                error_breakdown["Unknown Errors"] += 1
+
+        error_msg = ""
+        for key, value in error_breakdown.items():
+            if value > 0:
+                error_msg += "\n{}: {}\n".format(key, value)
+
+        return error_msg
+
+    async def outage_alerts(
+        self,
+        exception: APIError,
+        deployment_id: str,
+    ) -> None:
+        """
+        Send slack alert if model is badly configured / having an outage (408, 401, 429, >=500).
+
+        key = model_id
+
+        value = {
+        - model_id
+        - threshold
+        - alerts []
+        }
+
+        ttl = 1hr
+        max_alerts_size = 10
+        """
+        try:
+            outage_value: Optional[OutageModel] = await self.internal_usage_cache.async_get_cache(key=deployment_id)  # type: ignore
+            if (
+                getattr(exception, "status_code", None) is None
+                or (
+                    exception.status_code != 408  # type: ignore
+                    and exception.status_code < 500  # type: ignore
+                )
+                or self.llm_router is None
+            ):
+                return
+
+            ### EXTRACT MODEL DETAILS ###
+            deployment = self.llm_router.get_deployment(model_id=deployment_id)
+            if deployment is None:
+                return
+
+            model = deployment.litellm_params.model
+            provider = deployment.litellm_params.custom_llm_provider
+            if provider is None:
+                try:
+                    model, provider, _, _ = litellm.get_llm_provider(model=model)
+                except Exception as e:
+                    provider = ""
+            api_base = litellm.get_api_base(
+                model=model, optional_params=deployment.litellm_params
+            )
+
+            if outage_value is None:
+                outage_value = OutageModel(
+                    model_id=deployment_id,
+                    alerts=[exception.status_code],  # type: ignore
+                    deployment_ids=[deployment_id],
+                    minor_alert_sent=False,
+                    major_alert_sent=False,
+                    last_updated_at=time.time(),
+                )
+
+                ## add to cache ##
+                await self.internal_usage_cache.async_set_cache(
+                    key=deployment_id,
+                    value=outage_value,
+                    ttl=self.alerting_args.outage_alert_ttl,
+                )
+                return
+
+            outage_value["alerts"].append(exception.status_code)  # type: ignore
+            outage_value["deployment_ids"].append(deployment_id)
+            outage_value["last_updated_at"] = time.time()
+
+            ## MINOR OUTAGE ALERT SENT ##
+            if (
+                outage_value["minor_alert_sent"] == False
+                and len(outage_value["alerts"])
+                >= self.alerting_args.minor_outage_alert_threshold
+            ):
+                msg = f"""\n\n
+*⚠️ Minor Service Outage*
+
+*Model Name:* `{model}`
+*Provider:* `{provider}`
+*API Base:* `{api_base}`
+
+*Errors:*
+{self._count_outage_alerts(alerts=outage_value["alerts"])}
+
+
+*Last Check:* `{round(time.time() - outage_value["last_updated_at"], 4)}s ago`\n\n
+"""
+                # send minor alert
+                _result_val = self.send_alert(
+                    message=msg, level="Medium", alert_type="outage_alerts"
+                )
+                if _result_val is not None:
+                    await _result_val
+                # set to true
+                outage_value["minor_alert_sent"] = True
+            elif (
+                outage_value["major_alert_sent"] == False
+                and len(outage_value["alerts"])
+                >= self.alerting_args.major_outage_alert_threshold
+            ):
+                msg = f"""\n\n
+*⚠️ Major Service Outage*
+
+*Model Name:* `{model}`
+*Provider:* `{provider}`
+*API Base:* `{api_base}`
+
+*Errors:*
+{self._count_outage_alerts(alerts=outage_value["alerts"])}
+
+
+*Last Check:* `{round(time.time() - outage_value["last_updated_at"], 4)}s ago`\n\n
+"""
+                # send minor alert
+                await self.send_alert(
+                    message=msg, level="High", alert_type="outage_alerts"
+                )
+                # set to true
+                outage_value["major_alert_sent"] = True
+
+            ## update cache ##
+            await self.internal_usage_cache.async_set_cache(
+                key=deployment_id, value=outage_value
+            )
+        except Exception as e:
+            pass
+
     async def model_added_alert(
         self, model_name: str, litellm_model_name: str, passed_model_info: Any
     ):
@@ -750,10 +913,12 @@ Model Info:
 ```
 """
 
-        await self.send_alert(
+        alert_val = self.send_alert(
             message=message, level="Low", alert_type="new_model_added"
         )
-        pass
+
+        if alert_val is not None and asyncio.iscoroutine(alert_val):
+            await alert_val
 
     async def model_removed_alert(self, model_name: str):
         pass
@@ -948,6 +1113,7 @@ Model Info:
             "spend_reports",
             "new_model_added",
             "cooldown_deployment",
+            "outage_alerts",
         ],
         user_info: Optional[WebhookEvent] = None,
         **kwargs,
@@ -1071,10 +1237,12 @@ Model Info:
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         """Log failure + deployment latency"""
+        _litellm_params = kwargs.get("litellm_params", {})
+        _model_info = _litellm_params.get("model_info", {}) or {}
+        model_id = _model_info.get("id", "")
+        try:
             if "daily_reports" in self.alert_types:
-            model_id = (
+                try:
-                kwargs.get("litellm_params", {}).get("model_info", {}).get("id", "")
-            )
                     await self.async_update_daily_reports(
                         DeploymentMetrics(
                             id=model_id,
@@ -1083,6 +1251,39 @@ Model Info:
                             updated_at=litellm.utils.get_utc_datetime(),
                         )
                     )
+                except Exception as e:
+                    verbose_logger.debug(f"Exception raises -{str(e)}")
+
+            if "outage_alerts" in self.alert_types and isinstance(
+                kwargs.get("exception", ""), APIError
+            ):
+                _litellm_params = litellm.types.router.LiteLLM_Params(
+                    model=kwargs.get("model", ""),
+                    **kwargs.get("litellm_params", {}),
+                    **kwargs.get("optional_params", {}),
+                )
+                _region_name = litellm.utils._get_model_region(
+                    custom_llm_provider=kwargs.get("custom_llm_provider", ""),
+                    litellm_params=_litellm_params,
+                )
+                # if region name not known, default to api base #
+                if _region_name is None:
+                    _region_name = litellm.get_api_base(
+                        model=kwargs.get("model", ""),
+                        optional_params={
+                            **kwargs.get("litellm_params", {}),
+                            **kwargs.get("optional_params", {}),
+                        },
+                    )
+                    if _region_name is None:
+                        _region_name = ""
+
+                await self.outage_alerts(
+                    exception=kwargs["exception"],
+                    deployment_id=model_id,
+                )
+        except Exception as e:
+            pass
 
     async def _run_scheduler_helper(self, llm_router) -> bool:
         """

litellm/integrations/traceloop.py

@@ -1,29 +1,55 @@
+import traceback
+from litellm._logging import verbose_logger
+import litellm
+
+
 class TraceloopLogger:
     def __init__(self):
+        try:
             from traceloop.sdk.tracing.tracing import TracerWrapper
             from traceloop.sdk import Traceloop
+        except ModuleNotFoundError as e:
+            verbose_logger.error(
+                f"Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: {e}\n{traceback.format_exc()}"
+            )
 
-        Traceloop.init(app_name="Litellm-Server", disable_batch=True)
+        Traceloop.init(
+            app_name="Litellm-Server",
+            disable_batch=True,
+        )
         self.tracer_wrapper = TracerWrapper()
 
-    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+    def log_event(
-        from opentelemetry.trace import SpanKind
+        self,
+        kwargs,
+        response_obj,
+        start_time,
+        end_time,
+        user_id,
+        print_verbose,
+        level="DEFAULT",
+        status_message=None,
+    ):
+        from opentelemetry import trace
+        from opentelemetry.trace import SpanKind, Status, StatusCode
         from opentelemetry.semconv.ai import SpanAttributes
 
         try:
-            tracer = self.tracer_wrapper.get_tracer()
+            print_verbose(
+                f"Traceloop Logging - Enters logging function for model {kwargs}"
+            )
 
-            model = kwargs.get("model")
+            tracer = self.tracer_wrapper.get_tracer()
 
             # LiteLLM uses the standard OpenAI library, so it's already handled by Traceloop SDK
             if kwargs.get("litellm_params").get("custom_llm_provider") == "openai":
                 return
 
             optional_params = kwargs.get("optional_params", {})
-            with tracer.start_as_current_span(
+            span = tracer.start_span(
-                "litellm.completion",
+                "litellm.completion", kind=SpanKind.CLIENT, start_time=start_time
-                kind=SpanKind.CLIENT,
+            )
-            ) as span:
+
             if span.is_recording():
                 span.set_attribute(
                     SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")
@@ -50,9 +76,7 @@ class TraceloopLogger:
                 if "tools" in optional_params or "functions" in optional_params:
                     span.set_attribute(
                         SpanAttributes.LLM_REQUEST_FUNCTIONS,
-                            optional_params.get(
+                        optional_params.get("tools", optional_params.get("functions")),
-                                "tools", optional_params.get("functions")
-                            ),
                     )
                 if "user" in optional_params:
                     span.set_attribute(
@@ -65,7 +89,8 @@ class TraceloopLogger:
                     )
                 if "temperature" in optional_params:
                     span.set_attribute(
-                            SpanAttributes.LLM_TEMPERATURE, kwargs.get("temperature")
+                        SpanAttributes.LLM_REQUEST_TEMPERATURE,
+                        kwargs.get("temperature"),
                     )
 
                 for idx, prompt in enumerate(kwargs.get("messages")):
@@ -110,5 +135,15 @@ class TraceloopLogger:
                         choice.get("message").get("content"),
                     )
 
+            if (
+                level == "ERROR"
+                and status_message is not None
+                and isinstance(status_message, str)
+            ):
+                span.record_exception(Exception(status_message))
+                span.set_status(Status(StatusCode.ERROR, status_message))
+
+            span.end(end_time)
+
         except Exception as e:
             print_verbose(f"Traceloop Layer Error - {e}")

litellm/llms/prompt_templates/factory.py

@@ -126,7 +126,7 @@ def convert_to_ollama_image(openai_image_url: str):
         else:
             base64_data = openai_image_url
 
-        return base64_data;
+        return base64_data
     except Exception as e:
         if "Error: Unable to fetch image from URL" in str(e):
             raise e
@@ -134,6 +134,7 @@ def convert_to_ollama_image(openai_image_url: str):
             """Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{base64_image}". """
         )
 
+
 def ollama_pt(
     model, messages
 ):  # https://github.com/ollama/ollama/blob/af4cf55884ac54b9e637cd71dadfe9b7a5685877/docs/modelfile.md#template
@@ -166,7 +167,9 @@ def ollama_pt(
                         if element["type"] == "text":
                             prompt += element["text"]
                         elif element["type"] == "image_url":
-                            base64_image = convert_to_ollama_image(element["image_url"]["url"])
+                            base64_image = convert_to_ollama_image(
+                                element["image_url"]["url"]
+                            )
                             images.append(base64_image)
         return {"prompt": prompt, "images": images}
     else:
@@ -1533,6 +1536,7 @@ def _gemini_vision_convert_messages(messages: list):
                     # Case 2: Base64 image data
                     import base64
                     import io
+
                     # Extract the base64 image data
                     base64_data = img.split("base64,")[1]
 

litellm/main.py

@@ -420,6 +420,8 @@ def mock_completion(
                 api_key="mock-key",
             )
         if isinstance(mock_response, Exception):
+            if isinstance(mock_response, openai.APIError):
+                raise mock_response
             raise litellm.APIError(
                 status_code=500,  # type: ignore
                 message=str(mock_response),
@@ -463,7 +465,9 @@ def mock_completion(
 
         return model_response
 
-    except:
+    except Exception as e:
+        if isinstance(e, openai.APIError):
+            raise e
         traceback.print_exc()
         raise Exception("Mock completion response failed")
 
@@ -864,6 +868,7 @@ def completion(
             user=user,
             optional_params=optional_params,
             litellm_params=litellm_params,
+            custom_llm_provider=custom_llm_provider,
         )
         if mock_response:
             return mock_completion(

litellm/proxy/_experimental/out/_next/static/chunks/app/layout-235f9f140e0f64da.js

@@ -0,0 +1 @@
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{11837:function(n,e,t){Promise.resolve().then(t.t.bind(t,99646,23)),Promise.resolve().then(t.t.bind(t,63385,23))},63385:function(){},99646:function(n){n.exports={style:{fontFamily:"'__Inter_12bbc4', '__Inter_Fallback_12bbc4'",fontStyle:"normal"},className:"__className_12bbc4"}}},function(n){n.O(0,[971,69,744],function(){return n(n.s=11837)}),_N_E=n.O()}]);

litellm/proxy/_experimental/out/_next/static/chunks/app/layout-616577b9ef1fa19e.js

@@ -1 +0,0 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{93553:function(n,e,t){Promise.resolve().then(t.t.bind(t,63385,23)),Promise.resolve().then(t.t.bind(t,99646,23))},63385:function(){},99646:function(n){n.exports={style:{fontFamily:"'__Inter_12bbc4', '__Inter_Fallback_12bbc4'",fontStyle:"normal"},className:"__className_12bbc4"}}},function(n){n.O(0,[971,69,744],function(){return n(n.s=93553)}),_N_E=n.O()}]);

litellm/proxy/_experimental/out/_next/static/chunks/webpack-3cc604e175425ddd.js

@@ -1 +1 @@
-!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/103fe7af2014a1c2.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
+!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/9e367ab966b14e29.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();

litellm/proxy/_experimental/out/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-f7340db8b64cd999.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-f7340db8b64cd999.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/103fe7af2014a1c2.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[94430,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"507\",\"static/chunks/507-0aee992ad94e4137.js\",\"931\",\"static/chunks/app/page-9547f131c3870082.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/103fe7af2014a1c2.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"9Kn8POydvrC2EQ8cCUuvp\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_12bbc4\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
+<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-3cc604e175425ddd.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-3cc604e175425ddd.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/9e367ab966b14e29.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[94430,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"507\",\"static/chunks/507-0aee992ad94e4137.js\",\"931\",\"static/chunks/app/page-7219129f052f09c7.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/9e367ab966b14e29.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"x4G9dq-RO3w0GR_CKU40g\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_12bbc4\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>

litellm/proxy/_experimental/out/index.txt

@@ -1,7 +1,7 @@
 2:I[77831,[],""]
-3:I[94430,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","507","static/chunks/507-0aee992ad94e4137.js","931","static/chunks/app/page-9547f131c3870082.js"],""]
+3:I[94430,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","507","static/chunks/507-0aee992ad94e4137.js","931","static/chunks/app/page-7219129f052f09c7.js"],""]
 4:I[5613,[],""]
 5:I[31778,[],""]
-0:["9Kn8POydvrC2EQ8cCUuvp",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_12bbc4","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/103fe7af2014a1c2.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
+0:["x4G9dq-RO3w0GR_CKU40g",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_12bbc4","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/9e367ab966b14e29.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_super_secret_config.yaml

@@ -1,30 +1,47 @@
+general_settings:
+  alert_to_webhook_url:
+    budget_alerts: https://hooks.slack.com/services/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
+    daily_reports: https://hooks.slack.com/services/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
+    db_exceptions: https://hooks.slack.com/services/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
+    llm_exceptions: https://hooks.slack.com/services/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
+    llm_requests_hanging: https://hooks.slack.com/services/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
+    llm_too_slow: https://hooks.slack.com/services/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
+    outage_alerts: https://hooks.slack.com/services/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
+  alert_types:
+  - llm_exceptions
+  - llm_too_slow
+  - llm_requests_hanging
+  - budget_alerts
+  - db_exceptions
+  - daily_reports
+  - spend_reports
+  - cooldown_deployment
+  - new_model_added
+  - outage_alerts
+  alerting:
+  - slack
+  database_connection_pool_limit: 100
+  database_connection_timeout: 60
+  health_check_interval: 300
+  ui_access_mode: all
+litellm_settings:
+  json_logs: true
 model_list:
-  - model_name: gpt-3.5-turbo-fake-model
+- litellm_params:
-    litellm_params:
-      model: openai/my-fake-model
     api_base: http://0.0.0.0:8080
-      api_key: ""
+    api_key: ''
-  - model_name: gpt-3.5-turbo
+    model: openai/my-fake-model
-    litellm_params:
+  model_name: gpt-3.5-turbo-fake-model
-      model: azure/gpt-35-turbo
+- litellm_params:
     api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
     api_key: os.environ/AZURE_EUROPE_API_KEY
-  - model_name: gpt-3.5-turbo
+    model: azure/gpt-35-turbo
-    litellm_params:
+  model_name: gpt-3.5-turbo
-      model: azure/chatgpt-v-2
+- litellm_params:
     api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
-      api_version: "2023-05-15"
+    api_key: os.environ/AZURE_API_KEY
-      api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
+    api_version: '2023-05-15'
-  
+    model: azure/chatgpt-v-2
+  model_name: gpt-3.5-turbo
 router_settings:
   enable_pre_call_checks: true
-
-litellm_settings:
-  json_logs: True
-
-general_settings:
-  alerting: ["slack"]
-  alerting_args: 
-    report_check_interval: 10
-  enable_jwt_auth: True
-

litellm/proxy/_types.py

@@ -7,6 +7,19 @@ import uuid, json, sys, os
 from litellm.types.router import UpdateRouterConfig
 from litellm.types.utils import ProviderField
 
+AlertType = Literal[
+    "llm_exceptions",
+    "llm_too_slow",
+    "llm_requests_hanging",
+    "budget_alerts",
+    "db_exceptions",
+    "daily_reports",
+    "spend_reports",
+    "cooldown_deployment",
+    "new_model_added",
+    "outage_alerts",
+]
+
 
 def hash_token(token: str):
     import hashlib
@@ -855,17 +868,7 @@ class ConfigGeneralSettings(LiteLLMBase):
         None,
         description="List of alerting integrations. Today, just slack - `alerting: ['slack']`",
     )
-    alert_types: Optional[
+    alert_types: Optional[List[AlertType]] = Field(
-        List[
-            Literal[
-                "llm_exceptions",
-                "llm_too_slow",
-                "llm_requests_hanging",
-                "budget_alerts",
-                "db_exceptions",
-            ]
-        ]
-    ] = Field(
         None,
         description="List of alerting types. By default it is all alerts",
     )

litellm/proxy/proxy_server.py

@@ -484,9 +484,9 @@ async def user_api_key_auth(
             verbose_proxy_logger.debug("is_jwt: %s", is_jwt)
             if is_jwt:
                 # check if valid token
-                valid_token = await jwt_handler.auth_jwt(token=api_key)
+                jwt_valid_token: dict = await jwt_handler.auth_jwt(token=api_key)
                 # get scopes
-                scopes = jwt_handler.get_scopes(token=valid_token)
+                scopes = jwt_handler.get_scopes(token=jwt_valid_token)
 
                 # check if admin
                 is_admin = jwt_handler.is_admin(scopes=scopes)
@@ -509,7 +509,9 @@ async def user_api_key_auth(
                             f"Admin not allowed to access this route. Route={route}, Allowed Routes={actual_routes}"
                         )
                 # get team id
-                team_id = jwt_handler.get_team_id(token=valid_token, default_value=None)
+                team_id = jwt_handler.get_team_id(
+                    token=jwt_valid_token, default_value=None
+                )
 
                 if team_id is None and jwt_handler.is_required_team_id() == True:
                     raise Exception(
@@ -539,7 +541,9 @@ async def user_api_key_auth(
                     )
 
                 # [OPTIONAL] track spend for an org id - `LiteLLM_OrganizationTable`
-                org_id = jwt_handler.get_org_id(token=valid_token, default_value=None)
+                org_id = jwt_handler.get_org_id(
+                    token=jwt_valid_token, default_value=None
+                )
                 if org_id is not None:
                     _ = await get_org_object(
                         org_id=org_id,
@@ -548,7 +552,9 @@ async def user_api_key_auth(
                     )
                 # [OPTIONAL] track spend against an internal employee - `LiteLLM_UserTable`
                 user_object = None
-                user_id = jwt_handler.get_user_id(token=valid_token, default_value=None)
+                user_id = jwt_handler.get_user_id(
+                    token=jwt_valid_token, default_value=None
+                )
                 if user_id is not None:
                     # get the user object
                     user_object = await get_user_object(
@@ -561,7 +567,7 @@ async def user_api_key_auth(
                 # [OPTIONAL] track spend against an external user - `LiteLLM_EndUserTable`
                 end_user_object = None
                 end_user_id = jwt_handler.get_end_user_id(
-                    token=valid_token, default_value=None
+                    token=jwt_valid_token, default_value=None
                 )
                 if end_user_id is not None:
                     # get the end-user object
@@ -595,7 +601,7 @@ async def user_api_key_auth(
                             user_id=litellm_proxy_admin_name,
                             max_budget=litellm.max_budget,
                             spend=global_proxy_spend,
-                            token=valid_token["token"],
+                            token=jwt_valid_token["token"],
                         )
                         asyncio.create_task(
                             proxy_logging_obj.budget_alerts(
@@ -693,7 +699,9 @@ async def user_api_key_auth(
         ### CHECK IF ADMIN ###
         # note: never string compare api keys, this is vulenerable to a time attack. Use secrets.compare_digest instead
         ## Check CACHE
-        valid_token = user_api_key_cache.get_cache(key=hash_token(api_key))
+        valid_token: Optional[UserAPIKeyAuth] = user_api_key_cache.get_cache(
+            key=hash_token(api_key)
+        )
         if (
             valid_token is not None
             and isinstance(valid_token, UserAPIKeyAuth)
@@ -762,23 +770,19 @@ async def user_api_key_auth(
         original_api_key = api_key  # (Patch: For DynamoDB Backwards Compatibility)
         if api_key.startswith("sk-"):
             api_key = hash_token(token=api_key)
-        valid_token = user_api_key_cache.get_cache(key=api_key)
+        valid_token: Optional[UserAPIKeyAuth] = user_api_key_cache.get_cache(  # type: ignore
+            key=api_key
+        )
         if valid_token is None:
             ## check db
             verbose_proxy_logger.debug("api key: %s", api_key)
             if prisma_client is not None:
-                valid_token = await prisma_client.get_data(
+                _valid_token: Optional[BaseModel] = await prisma_client.get_data(
                     token=api_key, table_name="combined_view"
                 )
-            elif custom_db_client is not None:
+                if _valid_token is not None:
-                try:
+                    valid_token = UserAPIKeyAuth(
-                    valid_token = await custom_db_client.get_data(
+                        **_valid_token.model_dump(exclude_none=True)
-                        key=api_key, table_name="key"
-                    )
-                except:
-                    # (Patch: For DynamoDB Backwards Compatibility)
-                    valid_token = await custom_db_client.get_data(
-                        key=original_api_key, table_name="key"
                     )
             verbose_proxy_logger.debug("Token from db: %s", valid_token)
         elif valid_token is not None and isinstance(valid_token, UserAPIKeyAuth):
@@ -793,8 +797,8 @@ async def user_api_key_auth(
                 "allowed_model_region"
             )
 
-        user_id_information = None
+        user_id_information: Optional[List] = None
-        if valid_token:
+        if valid_token is not None:
             # Got Valid Token from Cache, DB
             # Run checks for
             # 1. If token can call model
@@ -915,16 +919,13 @@ async def user_api_key_auth(
                             table_name="user",
                             query_type="find_all",
                         )
+                        if user_id_information is not None:
                             for _id in user_id_information:
                                 await user_api_key_cache.async_set_cache(
                                     key=_id["user_id"],
                                     value=_id,
                                     ttl=UserAPIKeyCacheTTLEnum.user_information_cache.value,
                                 )
-                    if custom_db_client is not None:
-                        user_id_information = await custom_db_client.get_data(
-                            key=valid_token.user_id, table_name="user"
-                        )
 
                 verbose_proxy_logger.debug(
                     f"user_id_information: {user_id_information}"
@@ -1067,12 +1068,13 @@ async def user_api_key_auth(
                 # collect information for alerting #
                 ####################################
 
-                user_email = None
+                user_email: Optional[str] = None
                 # Check if the token has any user id information
                 if user_id_information is not None:
-                    if isinstance(user_id_information, list):
+                    specific_user_id_information = user_id_information[0]
-                        user_id_information = user_id_information[0]
+                    _user_email = specific_user_id_information.get("user_email", None)
-                    user_email = user_id_information.get("user_email", None)
+                    if _user_email is not None:
+                        user_email = str(_user_email)
 
                 call_info = CallInfo(
                     token=valid_token.token,
@@ -1229,24 +1231,11 @@ async def user_api_key_auth(
                 value=valid_token,
                 ttl=UserAPIKeyCacheTTLEnum.key_information_cache.value,
             )
-            valid_token_dict = _get_pydantic_json_dict(valid_token)
+            valid_token_dict = valid_token.model_dump(exclude_none=True)
             valid_token_dict.pop("token", None)
 
             if _end_user_object is not None:
                 valid_token_dict.update(end_user_params)
-            """
-            asyncio create task to update the user api key cache with the user db table as well
-
-            This makes the user row data accessible to pre-api call hooks.
-            """
-            if custom_db_client is not None:
-                asyncio.create_task(
-                    _cache_user_row(
-                        user_id=valid_token.user_id,
-                        cache=user_api_key_cache,
-                        db=custom_db_client,
-                    )
-                )
 
             if not _is_user_proxy_admin(user_id_information):  # if non-admin
                 if route in LiteLLMRoutes.openai_routes.value:
@@ -3026,7 +3015,7 @@ class ProxyConfig:
             general_settings["alert_types"] = _general_settings["alert_types"]
             proxy_logging_obj.alert_types = general_settings["alert_types"]
             proxy_logging_obj.slack_alerting_instance.update_values(
-                alert_types=general_settings["alert_types"]
+                alert_types=general_settings["alert_types"], llm_router=llm_router
             )
 
         if "alert_to_webhook_url" in _general_settings:
@@ -3034,7 +3023,8 @@ class ProxyConfig:
                 "alert_to_webhook_url"
             ]
             proxy_logging_obj.slack_alerting_instance.update_values(
-                alert_to_webhook_url=general_settings["alert_to_webhook_url"]
+                alert_to_webhook_url=general_settings["alert_to_webhook_url"],
+                llm_router=llm_router,
             )
 
     async def _update_general_settings(self, db_general_settings: Optional[Json]):
@@ -3602,6 +3592,9 @@ async def startup_event():
     ## Error Tracking ##
     error_tracking()
 
+    ## UPDATE SLACK ALERTING ##
+    proxy_logging_obj.slack_alerting_instance.update_values(llm_router=llm_router)
+
     db_writer_client = HTTPHandler()
 
     proxy_logging_obj._init_litellm_callbacks()  # INITIALIZE LITELLM CALLBACKS ON SERVER STARTUP <- do this to catch any logging errors on startup, not when calls are being made
@@ -9648,7 +9641,7 @@ async def google_login(request: Request):
             )
 
     ####### Detect DB + MASTER KEY in .env #######
-    if prisma_client is None and master_key is None:
+    if prisma_client is None or master_key is None:
         from fastapi.responses import HTMLResponse
 
         return HTMLResponse(content=missing_keys_html_form, status_code=200)
@@ -10775,6 +10768,9 @@ async def get_config():
                 }
             )
 
+        if llm_router is None:
+            _router_settings = {}
+        else:
             _router_settings = llm_router.get_settings()
         return {
             "status": "success",
@@ -10965,6 +10961,10 @@ async def health_services_endpoint(
                             test_message = f"Budget Alert test alert"
                         elif alert_type == "db_exceptions":
                             test_message = f"DB Exception test alert"
+                        elif alert_type == "outage_alerts":
+                            test_message = f"Outage Alert Exception test alert"
+                        elif alert_type == "daily_reports":
+                            test_message = f"Daily Reports test alert"
 
                         await proxy_logging_obj.alerting_handler(
                             message=test_message, level="Low", alert_type=alert_type

litellm/proxy/utils.py

@@ -13,6 +13,7 @@ from litellm.proxy._types import (
     Member,
     CallInfo,
     WebhookEvent,
+    AlertType,
 )
 from litellm.caching import DualCache, RedisCache
 from litellm.router import Deployment, ModelInfo, LiteLLM_Params
@@ -79,19 +80,7 @@ class ProxyLogging:
         self.cache_control_check = _PROXY_CacheControlCheck()
         self.alerting: Optional[List] = None
         self.alerting_threshold: float = 300  # default to 5 min. threshold
-        self.alert_types: List[
+        self.alert_types: List[AlertType] = [
-            Literal[
-                "llm_exceptions",
-                "llm_too_slow",
-                "llm_requests_hanging",
-                "budget_alerts",
-                "db_exceptions",
-                "daily_reports",
-                "spend_reports",
-                "cooldown_deployment",
-                "new_model_added",
-            ]
-        ] = [
             "llm_exceptions",
             "llm_too_slow",
             "llm_requests_hanging",
@@ -101,6 +90,7 @@ class ProxyLogging:
             "spend_reports",
             "cooldown_deployment",
             "new_model_added",
+            "outage_alerts",
         ]
         self.slack_alerting_instance = SlackAlerting(
             alerting_threshold=self.alerting_threshold,
@@ -114,21 +104,7 @@ class ProxyLogging:
         alerting: Optional[List],
         alerting_threshold: Optional[float],
         redis_cache: Optional[RedisCache],
-        alert_types: Optional[
+        alert_types: Optional[List[AlertType]] = None,
-            List[
-                Literal[
-                    "llm_exceptions",
-                    "llm_too_slow",
-                    "llm_requests_hanging",
-                    "budget_alerts",
-                    "db_exceptions",
-                    "daily_reports",
-                    "spend_reports",
-                    "cooldown_deployment",
-                    "new_model_added",
-                ]
-            ]
-        ] = None,
         alerting_args: Optional[dict] = None,
     ):
         self.alerting = alerting
@@ -2592,13 +2568,13 @@ def _is_valid_team_configs(team_id=None, team_config=None, request_data=None):
     return
 
 
-def _is_user_proxy_admin(user_id_information=None):
+def _is_user_proxy_admin(user_id_information: Optional[list]):
-    if (
+    if user_id_information is None:
-        user_id_information == None
-        or len(user_id_information) == 0
-        or user_id_information[0] == None
-    ):
         return False
+
+    if len(user_id_information) == 0 or user_id_information[0] is None:
+        return False
+
     _user = user_id_information[0]
     if (
         _user.get("user_role", None) is not None

litellm/router.py

@@ -3324,7 +3324,7 @@ class Router:
                     invalid_model_indices.append(idx)
                     continue
 
-            ## INVALID PARAMS ## -> catch 'gpt-3.5-turbo-16k' not supporting 'response_object' param
+            ## INVALID PARAMS ## -> catch 'gpt-3.5-turbo-16k' not supporting 'response_format' param
             if request_kwargs is not None and litellm.drop_params == False:
                 # get supported params
                 model, custom_llm_provider, _, _ = litellm.get_llm_provider(
@@ -3342,10 +3342,10 @@ class Router:
                     non_default_params = litellm.utils.get_non_default_params(
                         passed_params=request_kwargs
                     )
-                    special_params = ["response_object"]
+                    special_params = ["response_format"]
                     # check if all params are supported
                     for k, v in non_default_params.items():
-                        if k not in supported_openai_params:
+                        if k not in supported_openai_params and k in special_params:
                             # if not -> invalid model
                             verbose_router_logger.debug(
                                 f"INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k={k}"
@@ -3876,13 +3876,13 @@ class Router:
                 _api_base = litellm.get_api_base(
                     model=_model_name, optional_params=temp_litellm_params
                 )
-                asyncio.create_task(
+                # asyncio.create_task(
-                    proxy_logging_obj.slack_alerting_instance.send_alert(
+                #     proxy_logging_obj.slack_alerting_instance.send_alert(
-                        message=f"Router: Cooling down Deployment:\nModel Name: `{_model_name}`\nAPI Base: `{_api_base}`\nCooldown Time: `{cooldown_time} seconds`\nException Status Code: `{str(exception_status)}`\n\nChange 'cooldown_time' + 'allowed_fails' under 'Router Settings' on proxy UI, or via config - https://docs.litellm.ai/docs/proxy/reliability#fallbacks--retries--timeouts--cooldowns",
+                #         message=f"Router: Cooling down Deployment:\nModel Name: `{_model_name}`\nAPI Base: `{_api_base}`\nCooldown Time: `{cooldown_time} seconds`\nException Status Code: `{str(exception_status)}`\n\nChange 'cooldown_time' + 'allowed_fails' under 'Router Settings' on proxy UI, or via config - https://docs.litellm.ai/docs/proxy/reliability#fallbacks--retries--timeouts--cooldowns",
-                        alert_type="cooldown_deployment",
+                #         alert_type="cooldown_deployment",
-                        level="Low",
+                #         level="Low",
-                    )
+                #     )
-                )
+                # )
         except Exception as e:
             pass
 

litellm/tests/test_alerting.py

@@ -1,10 +1,11 @@
 # What is this?
 ## Tests slack alerting on proxy logging object
 
-import sys, json, uuid, random
+import sys, json, uuid, random, httpx
 import os
 import io, asyncio
 from datetime import datetime, timedelta
+from typing import Optional
 
 # import logging
 # logging.basicConfig(level=logging.DEBUG)
@@ -23,6 +24,7 @@ from unittest.mock import AsyncMock
 import pytest
 from litellm.router import AlertingConfig, Router
 from litellm.proxy._types import CallInfo
+from openai import APIError
 
 
 @pytest.mark.parametrize(
@@ -495,3 +497,109 @@ async def test_webhook_alerting(alerting_type):
                 user_info=user_info,
             )
         mock_send_alert.assert_awaited_once()
+
+
+@pytest.mark.parametrize(
+    "model, api_base, llm_provider, vertex_project, vertex_location",
+    [
+        ("gpt-3.5-turbo", None, "openai", None, None),
+        (
+            "azure/gpt-3.5-turbo",
+            "https://openai-gpt-4-test-v-1.openai.azure.com",
+            "azure",
+            None,
+            None,
+        ),
+        ("gemini-pro", None, "vertex_ai", "hardy-device-38811", "us-central1"),
+    ],
+)
+@pytest.mark.parametrize("error_code", [500, 408, 400])
+@pytest.mark.asyncio
+async def test_outage_alerting_called(
+    model, api_base, llm_provider, vertex_project, vertex_location, error_code
+):
+    """
+    If call fails, outage alert is called
+
+    If multiple calls fail, outage alert is sent
+    """
+    slack_alerting = SlackAlerting(alerting=["webhook"])
+
+    litellm.callbacks = [slack_alerting]
+
+    error_to_raise: Optional[APIError] = None
+
+    if error_code == 400:
+        print("RAISING 400 ERROR CODE")
+        error_to_raise = litellm.BadRequestError(
+            message="this is a bad request",
+            model=model,
+            llm_provider=llm_provider,
+        )
+    elif error_code == 408:
+        print("RAISING 408 ERROR CODE")
+        error_to_raise = litellm.Timeout(
+            message="A timeout occurred", model=model, llm_provider=llm_provider
+        )
+    elif error_code == 500:
+        print("RAISING 500 ERROR CODE")
+        error_to_raise = litellm.ServiceUnavailableError(
+            message="API is unavailable",
+            model=model,
+            llm_provider=llm_provider,
+            response=httpx.Response(
+                status_code=503,
+                request=httpx.Request(
+                    method="completion",
+                    url="https://github.com/BerriAI/litellm",
+                ),
+            ),
+        )
+
+    router = Router(
+        model_list=[
+            {
+                "model_name": model,
+                "litellm_params": {
+                    "model": model,
+                    "api_key": os.getenv("AZURE_API_KEY"),
+                    "api_base": api_base,
+                    "vertex_location": vertex_location,
+                    "vertex_project": vertex_project,
+                },
+            }
+        ],
+        num_retries=0,
+        allowed_fails=100,
+    )
+
+    slack_alerting.update_values(llm_router=router)
+    with patch.object(
+        slack_alerting, "outage_alerts", new=AsyncMock()
+    ) as mock_send_alert:
+        try:
+            await router.acompletion(
+                model=model,
+                messages=[{"role": "user", "content": "Hey!"}],
+                mock_response=error_to_raise,
+            )
+        except Exception as e:
+            pass
+
+        mock_send_alert.assert_called_once()
+
+    with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
+        for _ in range(3):
+            try:
+                await router.acompletion(
+                    model=model,
+                    messages=[{"role": "user", "content": "Hey!"}],
+                    mock_response=error_to_raise,
+                )
+            except Exception as e:
+                pass
+        await asyncio.sleep(3)
+        if error_code == 500 or error_code == 408:
+            mock_send_alert.assert_called_once()
+        else:
+            mock_send_alert.assert_not_called()

litellm/tests/test_traceloop.py

@@ -1,49 +1,35 @@
-# Commented out for now - since traceloop break ci/cd
+import sys
-# import sys
+import os
-# import os
+import time
-# import io, asyncio
+import pytest
+import litellm
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from traceloop.sdk import Traceloop
 
-# sys.path.insert(0, os.path.abspath('../..'))
+sys.path.insert(0, os.path.abspath("../.."))
-
-# from litellm import completion
-# import litellm
-# litellm.num_retries = 3
-# litellm.success_callback = [""]
-# import time
-# import pytest
-# from traceloop.sdk import Traceloop
-# Traceloop.init(app_name="test-litellm", disable_batch=True)
 
 
-# def test_traceloop_logging():
+@pytest.fixture()
-#     try:
+def exporter():
-#         litellm.set_verbose = True
+    exporter = InMemorySpanExporter()
-#         response = litellm.completion(
+    Traceloop.init(
-#             model="gpt-3.5-turbo",
+        app_name="test_litellm",
-#             messages=[{"role": "user", "content":"This is a test"}],
+        disable_batch=True,
-#             max_tokens=1000,
+        exporter=exporter,
-#             temperature=0.7,
+    )
-#             timeout=5,
+    litellm.success_callback = ["traceloop"]
-#         )
+    litellm.set_verbose = True
-#         print(f"response: {response}")
+
-#     except Exception as e:
+    return exporter
-#         pytest.fail(f"An exception occurred - {e}")
-# # test_traceloop_logging()
 
 
-# # def test_traceloop_logging_async():
+@pytest.mark.parametrize("model", ["claude-instant-1.2", "gpt-3.5-turbo"])
-# #     try:
+def test_traceloop_logging(exporter, model):
-# #         litellm.set_verbose = True
+
-# #         async def test_acompletion():
+    litellm.completion(
-# #             return await litellm.acompletion(
+        model=model,
-# #                 model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "This is a test"}],
-# #                 messages=[{"role": "user", "content":"This is a test"}],
+        max_tokens=1000,
-# #                 max_tokens=1000,
+        temperature=0.7,
-# #                 temperature=0.7,
+        timeout=5,
-# #                 timeout=5,
+    )
-# #             )
-# #         response = asyncio.run(test_acompletion())
-# #         print(f"response: {response}")
-# #     except Exception as e:
-# #         pytest.fail(f"An exception occurred - {e}")
-# # test_traceloop_logging_async()

litellm/utils.py

@@ -2027,6 +2027,7 @@ class Logging:
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
+                            user_id=kwargs.get("user", None),
                             print_verbose=print_verbose,
                         )
                     if callback == "s3":
@@ -2598,6 +2599,17 @@ class Logging:
                             level="ERROR",
                             kwargs=self.model_call_details,
                         )
+                    if callback == "traceloop":
+                        traceloopLogger.log_event(
+                            start_time=start_time,
+                            end_time=end_time,
+                            response_obj=None,
+                            user_id=kwargs.get("user", None),
+                            print_verbose=print_verbose,
+                            status_message=str(exception),
+                            level="ERROR",
+                            kwargs=self.model_call_details,
+                        )
                     if callback == "prometheus":
                         global prometheusLogger
                         verbose_logger.debug("reaches prometheus for success logging!")
@@ -6286,7 +6298,9 @@ def get_model_region(
     return None
 
 
-def get_api_base(model: str, optional_params: dict) -> Optional[str]:
+def get_api_base(
+    model: str, optional_params: Union[dict, LiteLLM_Params]
+) -> Optional[str]:
     """
     Returns the api base used for calling the model.
 
@@ -6306,7 +6320,9 @@ def get_api_base(model: str, optional_params: dict) -> Optional[str]:
     """
 
     try:
-        if "model" in optional_params:
+        if isinstance(optional_params, LiteLLM_Params):
+            _optional_params = optional_params
+        elif "model" in optional_params:
             _optional_params = LiteLLM_Params(**optional_params)
         else:  # prevent needing to copy and pop the dict
             _optional_params = LiteLLM_Params(
@@ -6699,6 +6715,8 @@ def get_llm_provider(
     Returns the provider for a given model name - e.g. 'azure/chatgpt-v-2' -> 'azure'
 
     For router -> Can also give the whole litellm param dict -> this function will extract the relevant details
+
+    Raises Error - if unable to map model to a provider
     """
     try:
         ## IF LITELLM PARAMS GIVEN ##
@@ -8632,7 +8650,16 @@ def exception_type(
                     )
                 elif hasattr(original_exception, "status_code"):
                     exception_mapping_worked = True
-                    if original_exception.status_code == 401:
+                    if original_exception.status_code == 400:
+                        exception_mapping_worked = True
+                        raise BadRequestError(
+                            message=f"{exception_provider} - {message}",
+                            llm_provider=custom_llm_provider,
+                            model=model,
+                            response=original_exception.response,
+                            litellm_debug_info=extra_information,
+                        )
+                    elif original_exception.status_code == 401:
                         exception_mapping_worked = True
                         raise AuthenticationError(
                             message=f"{exception_provider} - {message}",
@@ -9145,6 +9172,7 @@ def exception_type(
                             ),
                         ),
                     )
+
                 if hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 400:
                         exception_mapping_worked = True
@@ -9825,7 +9853,16 @@ def exception_type(
                     )
                 elif hasattr(original_exception, "status_code"):
                     exception_mapping_worked = True
-                    if original_exception.status_code == 401:
+                    if original_exception.status_code == 400:
+                        exception_mapping_worked = True
+                        raise BadRequestError(
+                            message=f"AzureException - {original_exception.message}",
+                            llm_provider="azure",
+                            model=model,
+                            litellm_debug_info=extra_information,
+                            response=original_exception.response,
+                        )
+                    elif original_exception.status_code == 401:
                         exception_mapping_worked = True
                         raise AuthenticationError(
                             message=f"AzureException - {original_exception.message}",
@@ -9842,7 +9879,7 @@ def exception_type(
                             litellm_debug_info=extra_information,
                             llm_provider="azure",
                         )
-                    if original_exception.status_code == 422:
+                    elif original_exception.status_code == 422:
                         exception_mapping_worked = True
                         raise BadRequestError(
                             message=f"AzureException - {original_exception.message}",

ui/litellm-dashboard/out/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-f7340db8b64cd999.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-f7340db8b64cd999.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/103fe7af2014a1c2.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[94430,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"507\",\"static/chunks/507-0aee992ad94e4137.js\",\"931\",\"static/chunks/app/page-9547f131c3870082.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/103fe7af2014a1c2.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"9Kn8POydvrC2EQ8cCUuvp\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_12bbc4\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
+<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-3cc604e175425ddd.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-3cc604e175425ddd.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/9e367ab966b14e29.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[94430,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"507\",\"static/chunks/507-0aee992ad94e4137.js\",\"931\",\"static/chunks/app/page-7219129f052f09c7.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/9e367ab966b14e29.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"x4G9dq-RO3w0GR_CKU40g\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_12bbc4\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>

ui/litellm-dashboard/out/index.txt

@@ -1,7 +1,7 @@
 2:I[77831,[],""]
-3:I[94430,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","507","static/chunks/507-0aee992ad94e4137.js","931","static/chunks/app/page-9547f131c3870082.js"],""]
+3:I[94430,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","507","static/chunks/507-0aee992ad94e4137.js","931","static/chunks/app/page-7219129f052f09c7.js"],""]
 4:I[5613,[],""]
 5:I[31778,[],""]
-0:["9Kn8POydvrC2EQ8cCUuvp",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_12bbc4","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/103fe7af2014a1c2.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
+0:["x4G9dq-RO3w0GR_CKU40g",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_12bbc4","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/9e367ab966b14e29.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

ui/litellm-dashboard/src/components/settings.tsx

@@ -24,7 +24,11 @@ import {
   Tab,
   Callout,
 } from "@tremor/react";
-import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
+import {
+  getCallbacksCall,
+  setCallbacksCall,
+  serviceHealthCheck,
+} from "./networking";
 import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
 import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
 
@@ -35,67 +39,69 @@ interface SettingsPageProps {
 }
 
 interface AlertingVariables {
-  SLACK_WEBHOOK_URL: string | null,
+  SLACK_WEBHOOK_URL: string | null;
-  LANGFUSE_PUBLIC_KEY: string | null, 
+  LANGFUSE_PUBLIC_KEY: string | null;
-  LANGFUSE_SECRET_KEY: string | null, 
+  LANGFUSE_SECRET_KEY: string | null;
-  LANGFUSE_HOST: string | null
+  LANGFUSE_HOST: string | null;
-  OPENMETER_API_KEY: string | null
+  OPENMETER_API_KEY: string | null;
 }
 
 interface AlertingObject {
-  name: string, 
+  name: string;
-  variables: AlertingVariables
+  variables: AlertingVariables;
 }
 
 const defaultLoggingObject: AlertingObject[] = [
   {
-    "name": "slack",
+    name: "slack",
-    "variables": {
+    variables: {
-      "LANGFUSE_HOST": null,
+      LANGFUSE_HOST: null,
-      "LANGFUSE_PUBLIC_KEY": null,
+      LANGFUSE_PUBLIC_KEY: null,
-      "LANGFUSE_SECRET_KEY": null,
+      LANGFUSE_SECRET_KEY: null,
-      "OPENMETER_API_KEY": null, 
+      OPENMETER_API_KEY: null,
-      "SLACK_WEBHOOK_URL": null
+      SLACK_WEBHOOK_URL: null,
-    }
+    },
   },
   {
-    "name": "langfuse",
+    name: "langfuse",
-    "variables": {
+    variables: {
-      "LANGFUSE_HOST": null,
+      LANGFUSE_HOST: null,
-      "LANGFUSE_PUBLIC_KEY": null,
+      LANGFUSE_PUBLIC_KEY: null,
-      "LANGFUSE_SECRET_KEY": null,
+      LANGFUSE_SECRET_KEY: null,
-      "OPENMETER_API_KEY": null, 
+      OPENMETER_API_KEY: null,
-      "SLACK_WEBHOOK_URL": null
+      SLACK_WEBHOOK_URL: null,
-    }
+    },
   },
   {
-    "name": "openmeter",
+    name: "openmeter",
-    "variables": {
+    variables: {
-      "LANGFUSE_HOST": null,
+      LANGFUSE_HOST: null,
-      "LANGFUSE_PUBLIC_KEY": null,
+      LANGFUSE_PUBLIC_KEY: null,
-      "LANGFUSE_SECRET_KEY": null,
+      LANGFUSE_SECRET_KEY: null,
-      "OPENMETER_API_KEY": null, 
+      OPENMETER_API_KEY: null,
-      "SLACK_WEBHOOK_URL": null
+      SLACK_WEBHOOK_URL: null,
-    }
+    },
-  }
+  },
-]
+];
 
 const Settings: React.FC<SettingsPageProps> = ({
   accessToken,
   userRole,
   userID,
 }) => {
-  const [callbacks, setCallbacks] = useState<AlertingObject[]>(defaultLoggingObject);
+  const [callbacks, setCallbacks] =
+    useState<AlertingObject[]>(defaultLoggingObject);
   const [alerts, setAlerts] = useState<any[]>([]);
   const [isModalVisible, setIsModalVisible] = useState(false);
   const [form] = Form.useForm();
   const [selectedCallback, setSelectedCallback] = useState<string | null>(null);
   const [selectedAlertValues, setSelectedAlertValues] = useState([]);
   const [catchAllWebhookURL, setCatchAllWebhookURL] = useState<string>("");
-  const [alertToWebhooks, setAlertToWebhooks] = useState<Record<string, string>>({});
+  const [alertToWebhooks, setAlertToWebhooks] = useState<
+    Record<string, string>
+  >({});
   const [activeAlerts, setActiveAlerts] = useState<string[]>([]);
 
-
   const handleSwitchChange = (alertName: string) => {
     if (activeAlerts.includes(alertName)) {
       setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName));
@@ -104,13 +110,14 @@ const Settings: React.FC<SettingsPageProps> = ({
     }
   };
   const alerts_to_UI_NAME: Record<string, string> = {
-    "llm_exceptions": "LLM Exceptions",
+    llm_exceptions: "LLM Exceptions",
-    "llm_too_slow": "LLM Responses Too Slow",
+    llm_too_slow: "LLM Responses Too Slow",
-    "llm_requests_hanging": "LLM Requests Hanging",
+    llm_requests_hanging: "LLM Requests Hanging",
-    "budget_alerts": "Budget Alerts (API Keys, Users)",
+    budget_alerts: "Budget Alerts (API Keys, Users)",
-    "db_exceptions": "Database Exceptions (Read/Write)",
+    db_exceptions: "Database Exceptions (Read/Write)",
-    "daily_reports": "Weekly/Monthly Spend Reports",
+    daily_reports: "Weekly/Monthly Spend Reports",
-  }
+    outage_alerts: "Outage Alerts",
+  };
 
   useEffect(() => {
     if (!accessToken || !userRole || !userID) {
@@ -121,15 +128,20 @@ const Settings: React.FC<SettingsPageProps> = ({
       let updatedCallbacks: any[] = defaultLoggingObject;
 
       updatedCallbacks = updatedCallbacks.map((item: any) => {
-        const callback = data.callbacks.find((cb: any) => cb.name === item.name);
+        const callback = data.callbacks.find(
+          (cb: any) => cb.name === item.name
+        );
         if (callback) {
-          return { ...item, variables: { ...item.variables, ...callback.variables } };
+          return {
+            ...item,
+            variables: { ...item.variables, ...callback.variables },
+          };
         } else {
           return item;
         }
       });
 
-      setCallbacks(updatedCallbacks)
+      setCallbacks(updatedCallbacks);
       // setCallbacks(callbacks_data);
 
       let alerts_data = data.alerts;
@@ -145,7 +157,6 @@ const Settings: React.FC<SettingsPageProps> = ({
           setActiveAlerts(active_alerts);
           setCatchAllWebhookURL(catch_all_webhook);
           setAlertToWebhooks(_alert_info.alerts_to_webhook);
-
         }
       }
 
@@ -153,10 +164,9 @@ const Settings: React.FC<SettingsPageProps> = ({
     });
   }, [accessToken, userRole, userID]);
 
-
   const isAlertOn = (alertName: string) => {
     return activeAlerts && activeAlerts.includes(alertName);
-  }
+  };
 
   const handleAddCallback = () => {
     console.log("Add callback clicked");
@@ -172,7 +182,7 @@ const Settings: React.FC<SettingsPageProps> = ({
   const handleChange = (values: any) => {
     setSelectedAlertValues(values);
     // Here, you can perform any additional logic with the selected values
-    console.log('Selected values:', values);
+    console.log("Selected values:", values);
   };
 
   const handleSaveAlerts = () => {
@@ -182,10 +192,12 @@ const Settings: React.FC<SettingsPageProps> = ({
 
     const updatedAlertToWebhooks: Record<string, string> = {};
     Object.entries(alerts_to_UI_NAME).forEach(([key, value]) => {
-      const webhookInput = document.querySelector(`input[name="${key}"]`) as HTMLInputElement;
+      const webhookInput = document.querySelector(
+        `input[name="${key}"]`
+      ) as HTMLInputElement;
       console.log("key", key);
       console.log("webhookInput", webhookInput);
-      const newWebhookValue = webhookInput?.value || '';
+      const newWebhookValue = webhookInput?.value || "";
       console.log("newWebhookValue", newWebhookValue);
       updatedAlertToWebhooks[key] = newWebhookValue;
     });
@@ -195,7 +207,7 @@ const Settings: React.FC<SettingsPageProps> = ({
     const payload = {
       general_settings: {
         alert_to_webhook_url: updatedAlertToWebhooks,
-        alert_types: activeAlerts
+        alert_types: activeAlerts,
       },
     };
 
@@ -204,10 +216,10 @@ const Settings: React.FC<SettingsPageProps> = ({
     try {
       setCallbacksCall(accessToken, payload);
     } catch (error) {
-      message.error('Failed to update alerts: ' + error, 20);
+      message.error("Failed to update alerts: " + error, 20);
     }
 
-    message.success('Alerts updated successfully');
+    message.success("Alerts updated successfully");
   };
   const handleSaveChanges = (callback: any) => {
     if (!accessToken) {
@@ -215,7 +227,11 @@ const Settings: React.FC<SettingsPageProps> = ({
     }
 
     const updatedVariables = Object.fromEntries(
-      Object.entries(callback.variables).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value])
+      Object.entries(callback.variables).map(([key, value]) => [
+        key,
+        (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)
+          ?.value || value,
+      ])
     );
 
     console.log("updatedVariables", updatedVariables);
@@ -224,8 +240,8 @@ const Settings: React.FC<SettingsPageProps> = ({
     const payload = {
       environment_variables: updatedVariables,
       litellm_settings: {
-        "success_callback": [callback.name]
+        success_callback: [callback.name],
-      }
+      },
     };
 
     try {
@@ -246,82 +262,82 @@ const Settings: React.FC<SettingsPageProps> = ({
       // Call API to add the callback
       console.log("Form values:", values);
       let payload;
-      if (values.callback === 'langfuse') {
+      if (values.callback === "langfuse") {
         payload = {
           environment_variables: {
             LANGFUSE_PUBLIC_KEY: values.langfusePublicKey,
-            LANGFUSE_SECRET_KEY: values.langfusePrivateKey
+            LANGFUSE_SECRET_KEY: values.langfusePrivateKey,
           },
           litellm_settings: {
-            success_callback: [values.callback]
+            success_callback: [values.callback],
-          }
+          },
         };
         setCallbacksCall(accessToken, payload);
         let newCallback: AlertingObject = {
-          "name": values.callback,
+          name: values.callback,
-          "variables": {
+          variables: {
-            "SLACK_WEBHOOK_URL": null,
+            SLACK_WEBHOOK_URL: null,
-            "LANGFUSE_HOST": null, 
+            LANGFUSE_HOST: null,
-            "LANGFUSE_PUBLIC_KEY": values.langfusePublicKey, 
+            LANGFUSE_PUBLIC_KEY: values.langfusePublicKey,
-            "LANGFUSE_SECRET_KEY": values.langfusePrivateKey,
+            LANGFUSE_SECRET_KEY: values.langfusePrivateKey,
-            OPENMETER_API_KEY: null
+            OPENMETER_API_KEY: null,
-          }
+          },
-        }
+        };
         // add langfuse to callbacks
         setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
-      } else if (values.callback === 'slack') {
+      } else if (values.callback === "slack") {
-        console.log(`values.slackWebhookUrl: ${values.slackWebhookUrl}`)
+        console.log(`values.slackWebhookUrl: ${values.slackWebhookUrl}`);
         payload = {
           general_settings: {
             alerting: ["slack"],
-            alerting_threshold: 300
+            alerting_threshold: 300,
           },
           environment_variables: {
-            SLACK_WEBHOOK_URL: values.slackWebhookUrl
+            SLACK_WEBHOOK_URL: values.slackWebhookUrl,
-          }
+          },
         };
         setCallbacksCall(accessToken, payload);
 
         // add slack to callbacks
-        console.log(`values.callback: ${values.callback}`)
+        console.log(`values.callback: ${values.callback}`);
 
         let newCallback: AlertingObject = {
-          "name": values.callback,
+          name: values.callback,
-          "variables": {
+          variables: {
-            "SLACK_WEBHOOK_URL": values.slackWebhookUrl,
+            SLACK_WEBHOOK_URL: values.slackWebhookUrl,
-            "LANGFUSE_HOST": null, 
+            LANGFUSE_HOST: null,
-            "LANGFUSE_PUBLIC_KEY": null, 
+            LANGFUSE_PUBLIC_KEY: null,
-            "LANGFUSE_SECRET_KEY": null, 
+            LANGFUSE_SECRET_KEY: null,
-            "OPENMETER_API_KEY": null
+            OPENMETER_API_KEY: null,
-          }
+          },
-        }
+        };
         setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
       } else if (values.callback == "openmeter") {
-        console.log(`values.openMeterApiKey: ${values.openMeterApiKey}`)
+        console.log(`values.openMeterApiKey: ${values.openMeterApiKey}`);
         payload = {
           environment_variables: {
             OPENMETER_API_KEY: values.openMeterApiKey,
           },
           litellm_settings: {
-            success_callback: [values.callback]
+            success_callback: [values.callback],
-          }
+          },
         };
         setCallbacksCall(accessToken, payload);
         let newCallback: AlertingObject = {
-          "name": values.callback,
+          name: values.callback,
-          "variables": {
+          variables: {
-            "SLACK_WEBHOOK_URL": null,
+            SLACK_WEBHOOK_URL: null,
-            "LANGFUSE_HOST": null, 
+            LANGFUSE_HOST: null,
-            "LANGFUSE_PUBLIC_KEY": null,
+            LANGFUSE_PUBLIC_KEY: null,
-            "LANGFUSE_SECRET_KEY": null,
+            LANGFUSE_SECRET_KEY: null,
-            OPENMETER_API_KEY: values.openMeterAPIKey
+            OPENMETER_API_KEY: values.openMeterAPIKey,
-          }
+          },
-        }
+        };
         // add langfuse to callbacks
         setCallbacks(callbacks ? [...callbacks, newCallback] : [newCallback]);
       } else {
         payload = {
-          error: 'Invalid callback value'
+          error: "Invalid callback value",
         };
       }
       setIsModalVisible(false);
@@ -338,13 +354,14 @@ const Settings: React.FC<SettingsPageProps> = ({
     return null;
   }
 
-  console.log(`callbacks: ${callbacks}`)
+  console.log(`callbacks: ${callbacks}`);
   return (
     <div className="w-full mx-4">
       <Grid numItems={1} className="gap-2 p-8 w-full mt-2">
-      <Callout title="[UI] Presidio PII + Guardrails Coming Soon. https://docs.litellm.ai/docs/proxy/pii_masking" color="sky">
+        <Callout
-
+          title="[UI] Presidio PII + Guardrails Coming Soon. https://docs.litellm.ai/docs/proxy/pii_masking"
-</Callout>
+          color="sky"
+        ></Callout>
         <TabGroup>
           <TabList variant="line" defaultValue="1">
             <Tab value="1">Logging Callbacks</Tab>
@@ -352,7 +369,6 @@ const Settings: React.FC<SettingsPageProps> = ({
           </TabList>
           <TabPanels>
             <TabPanel>
-          
               <Card>
                 <Table>
                   <TableHead>
@@ -362,29 +378,49 @@ const Settings: React.FC<SettingsPageProps> = ({
                     </TableRow>
                   </TableHead>
                   <TableBody>
-    {callbacks.filter((callback) => callback.name !== "slack").map((callback, index) => (
+                    {callbacks
+                      .filter((callback) => callback.name !== "slack")
+                      .map((callback, index) => (
                         <TableRow key={index}>
                           <TableCell>
                             <Badge color="emerald">{callback.name}</Badge>
                           </TableCell>
                           <TableCell>
                             <ul>
-            {Object.entries(callback.variables ?? {}).filter(([key, value]) => key.toLowerCase().includes(callback.name)).map(([key, value]) => (
+                              {Object.entries(callback.variables ?? {})
+                                .filter(([key, value]) =>
+                                  key.toLowerCase().includes(callback.name)
+                                )
+                                .map(([key, value]) => (
                                   <li key={key}>
                                     <Text className="mt-2">{key}</Text>
                                     {key === "LANGFUSE_HOST" ? (
-                <p>default value=https://cloud.langfuse.com</p>
+                                      <p>
+                                        default value=https://cloud.langfuse.com
+                                      </p>
                                     ) : (
                                       <div></div>
                                     )}
-              <TextInput name={key} defaultValue={value as string} type="password" />
+                                    <TextInput
+                                      name={key}
+                                      defaultValue={value as string}
+                                      type="password"
+                                    />
                                   </li>
                                 ))}
                             </ul>
-          <Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
+                            <Button
+                              className="mt-2"
+                              onClick={() => handleSaveChanges(callback)}
+                            >
                               Save Changes
                             </Button>
-          <Button onClick={() => serviceHealthCheck(accessToken, callback.name)} className="mx-2">
+                            <Button
+                              onClick={() =>
+                                serviceHealthCheck(accessToken, callback.name)
+                              }
+                              className="mx-2"
+                            >
                               Test Callback
                             </Button>
                           </TableCell>
@@ -392,14 +428,22 @@ const Settings: React.FC<SettingsPageProps> = ({
                       ))}
                   </TableBody>
                 </Table>
-              
               </Card>
             </TabPanel>
 
             <TabPanel>
-
               <Card>
-            <Text className="my-2">Alerts are only supported for Slack Webhook URLs. Get your webhook urls from <a href="https://api.slack.com/messaging/webhooks" target="_blank" style={{color: 'blue'}}>here</a></Text>
+                <Text className="my-2">
+                  Alerts are only supported for Slack Webhook URLs. Get your
+                  webhook urls from{" "}
+                  <a
+                    href="https://api.slack.com/messaging/webhooks"
+                    target="_blank"
+                    style={{ color: "blue" }}
+                  >
+                    here
+                  </a>
+                </Text>
                 <Table>
                   <TableHead>
                     <TableRow>
@@ -410,7 +454,8 @@ const Settings: React.FC<SettingsPageProps> = ({
                   </TableHead>
 
                   <TableBody>
-          {Object.entries(alerts_to_UI_NAME).map(([key, value], index) => (
+                    {Object.entries(alerts_to_UI_NAME).map(
+                      ([key, value], index) => (
                         <TableRow key={index}>
                           <TableCell>
                             <Switch
@@ -424,33 +469,35 @@ const Settings: React.FC<SettingsPageProps> = ({
                             <Text>{value}</Text>
                           </TableCell>
                           <TableCell>
-              <TextInput name={key} type="password" defaultValue={alertToWebhooks && alertToWebhooks[key] ? alertToWebhooks[key] : catchAllWebhookURL as string}>
+                            <TextInput
-                
+                              name={key}
-              </TextInput>
+                              type="password"
+                              defaultValue={
+                                alertToWebhooks && alertToWebhooks[key]
+                                  ? alertToWebhooks[key]
+                                  : (catchAllWebhookURL as string)
+                              }
+                            ></TextInput>
                           </TableCell>
                         </TableRow>
-          ))}
+                      )
+                    )}
                   </TableBody>
                 </Table>
                 <Button size="xs" className="mt-2" onClick={handleSaveAlerts}>
                   Save Changes
                 </Button>
 
-        <Button onClick={() => serviceHealthCheck(accessToken, "slack")} className="mx-2">
+                <Button
+                  onClick={() => serviceHealthCheck(accessToken, "slack")}
+                  className="mx-2"
+                >
                   Test Alerts
                 </Button>
-
-      
-
-
-
               </Card>
-            
             </TabPanel>
           </TabPanels>
         </TabGroup>
-
-      
       </Grid>
 
       <Modal
@@ -473,7 +520,7 @@ const Settings: React.FC<SettingsPageProps> = ({
             </Select>
           </Form.Item>
 
-          {selectedCallback === 'langfuse' && (
+          {selectedCallback === "langfuse" && (
             <>
               <Form.Item
                 label="LANGFUSE_PUBLIC_KEY"
@@ -497,19 +544,22 @@ const Settings: React.FC<SettingsPageProps> = ({
             </>
           )}
 
-          {
+          {selectedCallback == "openmeter" && (
-            selectedCallback == "openmeter" && <>
+            <>
               <Form.Item
                 label="OPENMETER_API_KEY"
                 name="openMeterApiKey"
                 rules={[
-                { required: true, message: "Please enter the openmeter api key" },
+                  {
+                    required: true,
+                    message: "Please enter the openmeter api key",
+                  },
                 ]}
               >
                 <TextInput type="password" />
               </Form.Item>
             </>
-          }
+          )}
 
           <div style={{ textAlign: "right", marginTop: "10px" }}>
             <Button2 htmlType="submit">Save</Button2>