phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix run sync fallbacks

Browse source
This commit is contained in:
Ishaan Jaff 2024-08-20 12:55:36 -07:00
parent e4b5e88a57
commit f6d97c25f2
2 changed files with 57 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

71
litellm/router.py
View file

@ -63,6 +63,7 @@ from litellm.router_utils.fallback_event_handlers import (
log_failure_fallback_event, log_failure_fallback_event,
log_success_fallback_event, log_success_fallback_event,
run_async_fallback, run_async_fallback,
run_sync_fallback,
) )
from litellm.router_utils.handle_error import send_llm_exception_alert from litellm.router_utils.handle_error import send_llm_exception_alert
from litellm.scheduler import FlowItem, Scheduler from litellm.scheduler import FlowItem, Scheduler
@ -2700,6 +2701,7 @@ class Router:
return response return response
except Exception as e: except Exception as e:
original_exception = e original_exception = e
original_model_group = kwargs.get("model")
verbose_router_logger.debug(f"An exception occurs {original_exception}") verbose_router_logger.debug(f"An exception occurs {original_exception}")
try: try:
verbose_router_logger.debug( verbose_router_logger.debug(
@ -2721,21 +2723,14 @@ class Router:
if fallback_model_group is None: if fallback_model_group is None:
raise original_exception raise original_exception
for mg in fallback_model_group: return run_sync_fallback(
""" *args,
Iterate through the model groups and try calling that deployment litellm_router=self,
""" fallback_model_group=fallback_model_group,
try: original_model_group=original_model_group,
## LOGGING original_exception=original_exception,
kwargs = self.log_retry(kwargs=kwargs, e=original_exception) **kwargs,
kwargs["model"] = mg )
kwargs.setdefault("metadata", {}).update(
{"model_group": mg}
) # update model_group used, if fallbacks are done
response = self.function_with_fallbacks(*args, **kwargs)
return response
except Exception as e:
pass
elif ( elif (
isinstance(e, litellm.ContentPolicyViolationError) isinstance(e, litellm.ContentPolicyViolationError)
and content_policy_fallbacks is not None and content_policy_fallbacks is not None
@ -2752,21 +2747,14 @@ class Router:
if fallback_model_group is None: if fallback_model_group is None:
raise original_exception raise original_exception
for mg in fallback_model_group: return run_sync_fallback(
""" *args,
Iterate through the model groups and try calling that deployment litellm_router=self,
""" fallback_model_group=fallback_model_group,
try: original_model_group=original_model_group,
## LOGGING original_exception=original_exception,
kwargs = self.log_retry(kwargs=kwargs, e=original_exception) **kwargs,
kwargs["model"] = mg )
kwargs.setdefault("metadata", {}).update(
{"model_group": mg}
) # update model_group used, if fallbacks are done
response = self.function_with_fallbacks(*args, **kwargs)
return response
except Exception as e:
pass
elif fallbacks is not None: elif fallbacks is not None:
verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
fallback_model_group = None fallback_model_group = None
@ -2790,21 +2778,14 @@ class Router:
if fallback_model_group is None: if fallback_model_group is None:
raise original_exception raise original_exception
for mg in fallback_model_group: return run_sync_fallback(
""" *args,
Iterate through the model groups and try calling that deployment litellm_router=self,
""" fallback_model_group=fallback_model_group,
try: original_model_group=original_model_group,
## LOGGING original_exception=original_exception,
kwargs = self.log_retry(kwargs=kwargs, e=original_exception) **kwargs,
kwargs["model"] = mg )
kwargs.setdefault("metadata", {}).update(
{"model_group": mg}
) # update model_group used, if fallbacks are done
response = self.function_with_fallbacks(*args, **kwargs)
return response
except Exception as e:
raise e
except Exception as e: except Exception as e:
raise e raise e
raise original_exception raise original_exception

31
litellm/router_utils/fallback_event_handlers.py
View file

@ -52,6 +52,37 @@ async def run_async_fallback(
raise error_from_fallbacks raise error_from_fallbacks
def run_sync_fallback(
litellm_router: LitellmRouter,
*args: Tuple[Any],
fallback_model_group: List[str],
original_model_group: str,
original_exception: Exception,
**kwargs,
) -> Any:
"""
Iterate through the model groups and try calling that deployment.
"""
error_from_fallbacks = original_exception
for mg in fallback_model_group:
if mg == original_model_group:
continue
try:
# LOGGING
kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
verbose_router_logger.info(f"Falling back to model_group = {mg}")
kwargs["model"] = mg
kwargs.setdefault("metadata", {}).update(
{"model_group": mg}
) # update model_group used, if fallbacks are done
response = litellm_router.function_with_fallbacks(*args, **kwargs)
verbose_router_logger.info("Successful fallback b/w models.")
return response
except Exception as e:
error_from_fallbacks = e
raise error_from_fallbacks
async def log_success_fallback_event(original_model_group: str, kwargs: dict): async def log_success_fallback_event(original_model_group: str, kwargs: dict):
for _callback in litellm.callbacks: for _callback in litellm.callbacks:
if isinstance(_callback, CustomLogger): if isinstance(_callback, CustomLogger):