phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) improve logging when no fallbacks found

Browse source
This commit is contained in:
ishaan-jaff 2024-01-08 08:53:40 +05:30
parent 7742950c57
commit ccd100fab3
1 changed files with 24 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

42
litellm/router.py
View file

@ -353,15 +353,15 @@ class Router:
model_client = potential_model_client model_client = potential_model_client
self.total_calls[model_name] += 1 self.total_calls[model_name] += 1
response = await litellm.acompletion( response = await litellm.acompletion(
**{ **{
**data, **data,
"messages": messages, "messages": messages,
"caching": self.cache_responses, "caching": self.cache_responses,
"client": model_client, "client": model_client,
"timeout": self.timeout, "timeout": self.timeout,
**kwargs, **kwargs,
} }
) )
self.success_calls[model_name] += 1 self.success_calls[model_name] += 1
return response return response
except Exception as e: except Exception as e:
@ -613,15 +613,15 @@ class Router:
model_client = potential_model_client model_client = potential_model_client
self.total_calls[model_name] += 1 self.total_calls[model_name] += 1
response = await litellm.atext_completion( response = await litellm.atext_completion(
**{ **{
**data, **data,
"prompt": prompt, "prompt": prompt,
"caching": self.cache_responses, "caching": self.cache_responses,
"client": model_client, "client": model_client,
"timeout": self.timeout, "timeout": self.timeout,
**kwargs, **kwargs,
} }
) )
self.success_calls[model_name] += 1 self.success_calls[model_name] += 1
return response return response
except Exception as e: except Exception as e:
@ -768,6 +768,7 @@ class Router:
f"An exception occurs: {e}\n\n Traceback{traceback.format_exc()}" f"An exception occurs: {e}\n\n Traceback{traceback.format_exc()}"
) )
original_exception = e original_exception = e
fallback_model_group = None
try: try:
if ( if (
hasattr(e, "status_code") and e.status_code == 400 hasattr(e, "status_code") and e.status_code == 400
@ -807,6 +808,11 @@ class Router:
if list(item.keys())[0] == model_group: if list(item.keys())[0] == model_group:
fallback_model_group = item[model_group] fallback_model_group = item[model_group]
break break
if fallback_model_group is None:
self.print_verbose(
f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}"
)
raise original_exception
for mg in fallback_model_group: for mg in fallback_model_group:
""" """
Iterate through the model groups and try calling that deployment Iterate through the model groups and try calling that deployment