From 276041e3bbe7c68485d4d4432195313551eedaee Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 23 Nov 2023 11:27:14 -0800 Subject: [PATCH] docs(routing.md): updating routing docs to include cooldown info --- docs/my-website/docs/routing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index cc4f7b548b..d69e32295a 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -8,6 +8,9 @@ import TabItem from '@theme/TabItem'; LiteLLM manages: - Load-balance across multiple deployments (e.g. Azure/OpenAI) - Prioritizing important requests to ensure they don't fail (i.e. Queueing) +- Basic reliability logic - cooldowns, fallbacks, timeouts and retries (fixed + exponential backoff) across multiple deployments/providers. + +In production, litellm supports using Redis as a way to track cooldown server and usage (managing tpm/rpm limits). ## Load Balancing (s/o [@paulpierre](https://www.linkedin.com/in/paulpierre/) for his contribution to this implementation)