phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(router testing) Add testing coverage for run_async_fallback and run_sync_fallback (#6256)

Browse source 
* add type hints for run_async_fallback

* fix async fallback doc string

* test run_async_fallback
This commit is contained in:
Ishaan Jaff 2024-10-16 16:16:17 +05:30 • committed by GitHub
parent d9a71650e3
commit fc5b75d171
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 388 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

39
litellm/router_utils/fallback_event_handlers.py
View file

@ -22,7 +22,24 @@ async def run_async_fallback(
**kwargs,
) -> Any:
"""
Iterate through the model groups and try calling that deployment.
Loops through all the fallback model groups and calls kwargs["original_function"] with the arguments and keyword arguments provided.
If the call is successful, it logs the success and returns the response.
If the call fails, it logs the failure and continues to the next fallback model group.
If all fallback model groups fail, it raises the most recent exception.
Args:
litellm_router: The litellm router instance.
*args: Positional arguments.
fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
original_model_group: The original model group. example: "gpt-3.5-turbo"
original_exception: The original exception.
**kwargs: Keyword arguments.
Returns:
The response from the successful fallback model group.
Raises:
The most recent exception if all fallback model groups fail.
"""
error_from_fallbacks = original_exception
for mg in fallback_model_group:
@ -66,7 +83,25 @@ def run_sync_fallback(
**kwargs,
) -> Any:
"""
Iterate through the fallback model groups and try calling each fallback deployment.
Synchronous version of run_async_fallback.
Loops through all the fallback model groups and calls kwargs["original_function"] with the arguments and keyword arguments provided.
If the call is successful, returns the response.
If the call fails, continues to the next fallback model group.
If all fallback model groups fail, it raises the most recent exception.
Args:
litellm_router: The litellm router instance.
*args: Positional arguments.
fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
original_model_group: The original model group. example: "gpt-3.5-turbo"
original_exception: The original exception.
**kwargs: Keyword arguments.
Returns:
The response from the successful fallback model group.
Raises:
The most recent exception if all fallback model groups fail.
"""
error_from_fallbacks = original_exception
for mg in fallback_model_group: