forked from phoenix/litellm-mirror
add context window exceeded error tutorial
This commit is contained in:
parent
38564ddc82
commit
879f97f620
1 changed files with 30 additions and 2 deletions
|
@ -1,4 +1,4 @@
|
||||||
# Basic Model Fallbacks w/ LiteLLM
|
# Model Fallbacks w/ LiteLLM
|
||||||
|
|
||||||
Here's how you can implement model fallbacks across 3 LLM providers (OpenAI, Anthropic, Azure) using LiteLLM.
|
Here's how you can implement model fallbacks across 3 LLM providers (OpenAI, Anthropic, Azure) using LiteLLM.
|
||||||
|
|
||||||
|
@ -7,7 +7,7 @@ Here's how you can implement model fallbacks across 3 LLM providers (OpenAI, Ant
|
||||||
!pip install litellm
|
!pip install litellm
|
||||||
```
|
```
|
||||||
|
|
||||||
## 2. Complete Code
|
## 2. Basic Fallbacks Code
|
||||||
```python
|
```python
|
||||||
import litellm
|
import litellm
|
||||||
from litellm import embedding, completion
|
from litellm import embedding, completion
|
||||||
|
@ -29,4 +29,32 @@ for model in model_fallback_list:
|
||||||
response = completion(model=model, messages=messages)
|
response = completion(model=model, messages=messages)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"error occurred: {traceback.format_exc()}")
|
print(f"error occurred: {traceback.format_exc()}")
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Context Window Exceptions
|
||||||
|
LiteLLM provides a sub-class of the InvalidRequestError class for Context Window Exceeded errors ([docs](https://docs.litellm.ai/docs/exception_mapping)).
|
||||||
|
|
||||||
|
Implement model fallbacks based on context window exceptions.
|
||||||
|
|
||||||
|
```python
|
||||||
|
import litellm
|
||||||
|
from litellm import completion, ContextWindowExceededError, completion_with_fallbacks
|
||||||
|
|
||||||
|
# set ENV variables
|
||||||
|
os.environ["OPENAI_API_KEY"] = ""
|
||||||
|
os.environ["ANTHROPIC_API_KEY"] = ""
|
||||||
|
os.environ["AZURE_API_KEY"] = ""
|
||||||
|
os.environ["AZURE_API_BASE"] = ""
|
||||||
|
os.environ["AZURE_API_VERSION"] = ""
|
||||||
|
|
||||||
|
context_window_fallback_list = ["gpt-3.5-turbo-16k", "gpt-4", "claude-instant-1"]
|
||||||
|
|
||||||
|
user_message = "Hello, how are you?"
|
||||||
|
messages = [{ "content": user_message,"role": "user"}]
|
||||||
|
|
||||||
|
for model in context_window_fallback_list:
|
||||||
|
try:
|
||||||
|
response = completion(model=model, messages=messages)
|
||||||
|
except ContextWindowExceededError as e:
|
||||||
|
completion_with_fallbacks(model=context_window_fallback_list[0], messages=messages, fallbacks=context_window_fallback_list[1:])
|
||||||
```
|
```
|
Loading…
Add table
Add a link
Reference in a new issue