From 1f76b0e7217c48ef95e91617c2b8842b0fbab18a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 29 Dec 2023 15:22:12 +0530 Subject: [PATCH] docs(routing.md): add retry_after to docs --- docs/my-website/docs/routing.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 3334dbd5c..8c58c10e6 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -251,6 +251,25 @@ response = router.completion(model="gpt-3.5-turbo", messages=messages) print(f"response: {response}") ``` +We also support setting minimum time to wait before retrying a failed request. This is via the `retry_after` param. + +```python +from litellm import Router + +model_list = [{...}] + +router = Router(model_list=model_list, + num_retries=3, retry_after=5) # waits min 5s before retrying request + +user_message = "Hello, whats the weather in San Francisco??" +messages = [{"content": user_message, "role": "user"}] + +# normal call +response = router.completion(model="gpt-3.5-turbo", messages=messages) + +print(f"response: {response}") +``` + ### Fallbacks If a call fails after num_retries, fall back to another model group.