mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 10:44:24 +00:00
* LiteLLM Minor Fixes & Improvements (09/23/2024) (#5842) * feat(auth_utils.py): enable admin to allow client-side credentials to be passed Makes it easier for devs to experiment with finetuned fireworks ai models * feat(router.py): allow setting configurable_clientside_auth_params for a model Closes https://github.com/BerriAI/litellm/issues/5843 * build(model_prices_and_context_window.json): fix anthropic claude-3-5-sonnet max output token limit Fixes https://github.com/BerriAI/litellm/issues/5850 * fix(azure_ai/): support content list for azure ai Fixes https://github.com/BerriAI/litellm/issues/4237 * fix(litellm_logging.py): always set saved_cache_cost Set to 0 by default * fix(fireworks_ai/cost_calculator.py): add fireworks ai default pricing handles calling 405b+ size models * fix(slack_alerting.py): fix error alerting for failed spend tracking Fixes regression with slack alerting error monitoring * fix(vertex_and_google_ai_studio_gemini.py): handle gemini no candidates in streaming chunk error * docs(bedrock.md): add llama3-1 models * test: fix tests * fix(azure_ai/chat): fix transformation for azure ai calls
This commit is contained in:
Krish Dholakia
GitHub
parent
4df9aca45e
commit
d37c8b5c6b
25 changed files with 611 additions and 294 deletions
|
|
@ -4335,157 +4335,177 @@ class Router:
|
|||
|
||||
total_tpm: Optional[int] = None
|
||||
total_rpm: Optional[int] = None
|
||||
configurable_clientside_auth_params: Optional[List[str]] = None
|
||||
|
||||
for model in self.model_list:
|
||||
if "model_name" in model and model["model_name"] == model_group:
|
||||
# model in model group found #
|
||||
litellm_params = LiteLLM_Params(**model["litellm_params"])
|
||||
# get model tpm
|
||||
_deployment_tpm: Optional[int] = None
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = model.get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = model.get("litellm_params", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = model.get("model_info", {}).get("tpm", None)
|
||||
is_match = False
|
||||
if (
|
||||
"model_name" in model and model["model_name"] == model_group
|
||||
): # exact match
|
||||
is_match = True
|
||||
elif (
|
||||
"model_name" in model
|
||||
and model_group in self.provider_default_deployments
|
||||
): # wildcard model
|
||||
is_match = True
|
||||
|
||||
if _deployment_tpm is not None:
|
||||
if total_tpm is None:
|
||||
total_tpm = 0
|
||||
total_tpm += _deployment_tpm # type: ignore
|
||||
# get model rpm
|
||||
_deployment_rpm: Optional[int] = None
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = model.get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = model.get("litellm_params", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = model.get("model_info", {}).get("rpm", None)
|
||||
if not is_match:
|
||||
continue
|
||||
# model in model group found #
|
||||
litellm_params = LiteLLM_Params(**model["litellm_params"])
|
||||
# get configurable clientside auth params
|
||||
configurable_clientside_auth_params = (
|
||||
litellm_params.configurable_clientside_auth_params
|
||||
)
|
||||
# get model tpm
|
||||
_deployment_tpm: Optional[int] = None
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = model.get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = model.get("litellm_params", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = model.get("model_info", {}).get("tpm", None)
|
||||
|
||||
if _deployment_rpm is not None:
|
||||
if total_rpm is None:
|
||||
total_rpm = 0
|
||||
total_rpm += _deployment_rpm # type: ignore
|
||||
# get model info
|
||||
try:
|
||||
model_info = litellm.get_model_info(model=litellm_params.model)
|
||||
except Exception:
|
||||
model_info = None
|
||||
# get llm provider
|
||||
model, llm_provider = "", ""
|
||||
try:
|
||||
model, llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=litellm_params.model,
|
||||
custom_llm_provider=litellm_params.custom_llm_provider,
|
||||
if _deployment_tpm is not None:
|
||||
if total_tpm is None:
|
||||
total_tpm = 0
|
||||
total_tpm += _deployment_tpm # type: ignore
|
||||
# get model rpm
|
||||
_deployment_rpm: Optional[int] = None
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = model.get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = model.get("litellm_params", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = model.get("model_info", {}).get("rpm", None)
|
||||
|
||||
if _deployment_rpm is not None:
|
||||
if total_rpm is None:
|
||||
total_rpm = 0
|
||||
total_rpm += _deployment_rpm # type: ignore
|
||||
# get model info
|
||||
try:
|
||||
model_info = litellm.get_model_info(model=litellm_params.model)
|
||||
except Exception:
|
||||
model_info = None
|
||||
# get llm provider
|
||||
model, llm_provider = "", ""
|
||||
try:
|
||||
model, llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=litellm_params.model,
|
||||
custom_llm_provider=litellm_params.custom_llm_provider,
|
||||
)
|
||||
except litellm.exceptions.BadRequestError as e:
|
||||
verbose_router_logger.error(
|
||||
"litellm.router.py::get_model_group_info() - {}".format(str(e))
|
||||
)
|
||||
|
||||
if model_info is None:
|
||||
supported_openai_params = litellm.get_supported_openai_params(
|
||||
model=model, custom_llm_provider=llm_provider
|
||||
)
|
||||
if supported_openai_params is None:
|
||||
supported_openai_params = []
|
||||
model_info = ModelMapInfo(
|
||||
key=model_group,
|
||||
max_tokens=None,
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
input_cost_per_token=0,
|
||||
output_cost_per_token=0,
|
||||
litellm_provider=llm_provider,
|
||||
mode="chat",
|
||||
supported_openai_params=supported_openai_params,
|
||||
supports_system_messages=None,
|
||||
)
|
||||
|
||||
if model_group_info is None:
|
||||
model_group_info = ModelGroupInfo(
|
||||
model_group=user_facing_model_group_name, providers=[llm_provider], **model_info # type: ignore
|
||||
)
|
||||
else:
|
||||
# if max_input_tokens > curr
|
||||
# if max_output_tokens > curr
|
||||
# if input_cost_per_token > curr
|
||||
# if output_cost_per_token > curr
|
||||
# supports_parallel_function_calling == True
|
||||
# supports_vision == True
|
||||
# supports_function_calling == True
|
||||
if llm_provider not in model_group_info.providers:
|
||||
model_group_info.providers.append(llm_provider)
|
||||
if (
|
||||
model_info.get("max_input_tokens", None) is not None
|
||||
and model_info["max_input_tokens"] is not None
|
||||
and (
|
||||
model_group_info.max_input_tokens is None
|
||||
or model_info["max_input_tokens"]
|
||||
> model_group_info.max_input_tokens
|
||||
)
|
||||
except litellm.exceptions.BadRequestError as e:
|
||||
verbose_router_logger.error(
|
||||
"litellm.router.py::get_model_group_info() - {}".format(str(e))
|
||||
):
|
||||
model_group_info.max_input_tokens = model_info["max_input_tokens"]
|
||||
if (
|
||||
model_info.get("max_output_tokens", None) is not None
|
||||
and model_info["max_output_tokens"] is not None
|
||||
and (
|
||||
model_group_info.max_output_tokens is None
|
||||
or model_info["max_output_tokens"]
|
||||
> model_group_info.max_output_tokens
|
||||
)
|
||||
):
|
||||
model_group_info.max_output_tokens = model_info["max_output_tokens"]
|
||||
if model_info.get("input_cost_per_token", None) is not None and (
|
||||
model_group_info.input_cost_per_token is None
|
||||
or model_info["input_cost_per_token"]
|
||||
> model_group_info.input_cost_per_token
|
||||
):
|
||||
model_group_info.input_cost_per_token = model_info[
|
||||
"input_cost_per_token"
|
||||
]
|
||||
if model_info.get("output_cost_per_token", None) is not None and (
|
||||
model_group_info.output_cost_per_token is None
|
||||
or model_info["output_cost_per_token"]
|
||||
> model_group_info.output_cost_per_token
|
||||
):
|
||||
model_group_info.output_cost_per_token = model_info[
|
||||
"output_cost_per_token"
|
||||
]
|
||||
if (
|
||||
model_info.get("supports_parallel_function_calling", None)
|
||||
is not None
|
||||
and model_info["supports_parallel_function_calling"] is True # type: ignore
|
||||
):
|
||||
model_group_info.supports_parallel_function_calling = True
|
||||
if (
|
||||
model_info.get("supports_vision", None) is not None
|
||||
and model_info["supports_vision"] is True # type: ignore
|
||||
):
|
||||
model_group_info.supports_vision = True
|
||||
if (
|
||||
model_info.get("supports_function_calling", None) is not None
|
||||
and model_info["supports_function_calling"] is True # type: ignore
|
||||
):
|
||||
model_group_info.supports_function_calling = True
|
||||
if (
|
||||
model_info.get("supported_openai_params", None) is not None
|
||||
and model_info["supported_openai_params"] is not None
|
||||
):
|
||||
model_group_info.supported_openai_params = model_info[
|
||||
"supported_openai_params"
|
||||
]
|
||||
|
||||
if model_info is None:
|
||||
supported_openai_params = litellm.get_supported_openai_params(
|
||||
model=model, custom_llm_provider=llm_provider
|
||||
)
|
||||
if supported_openai_params is None:
|
||||
supported_openai_params = []
|
||||
model_info = ModelMapInfo(
|
||||
key=model_group,
|
||||
max_tokens=None,
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
input_cost_per_token=0,
|
||||
output_cost_per_token=0,
|
||||
litellm_provider=llm_provider,
|
||||
mode="chat",
|
||||
supported_openai_params=supported_openai_params,
|
||||
supports_system_messages=None,
|
||||
)
|
||||
if model_group_info is not None:
|
||||
## UPDATE WITH TOTAL TPM/RPM FOR MODEL GROUP
|
||||
if total_tpm is not None:
|
||||
model_group_info.tpm = total_tpm
|
||||
|
||||
if model_group_info is None:
|
||||
model_group_info = ModelGroupInfo(
|
||||
model_group=user_facing_model_group_name, providers=[llm_provider], **model_info # type: ignore
|
||||
)
|
||||
else:
|
||||
# if max_input_tokens > curr
|
||||
# if max_output_tokens > curr
|
||||
# if input_cost_per_token > curr
|
||||
# if output_cost_per_token > curr
|
||||
# supports_parallel_function_calling == True
|
||||
# supports_vision == True
|
||||
# supports_function_calling == True
|
||||
if llm_provider not in model_group_info.providers:
|
||||
model_group_info.providers.append(llm_provider)
|
||||
if (
|
||||
model_info.get("max_input_tokens", None) is not None
|
||||
and model_info["max_input_tokens"] is not None
|
||||
and (
|
||||
model_group_info.max_input_tokens is None
|
||||
or model_info["max_input_tokens"]
|
||||
> model_group_info.max_input_tokens
|
||||
)
|
||||
):
|
||||
model_group_info.max_input_tokens = model_info[
|
||||
"max_input_tokens"
|
||||
]
|
||||
if (
|
||||
model_info.get("max_output_tokens", None) is not None
|
||||
and model_info["max_output_tokens"] is not None
|
||||
and (
|
||||
model_group_info.max_output_tokens is None
|
||||
or model_info["max_output_tokens"]
|
||||
> model_group_info.max_output_tokens
|
||||
)
|
||||
):
|
||||
model_group_info.max_output_tokens = model_info[
|
||||
"max_output_tokens"
|
||||
]
|
||||
if model_info.get("input_cost_per_token", None) is not None and (
|
||||
model_group_info.input_cost_per_token is None
|
||||
or model_info["input_cost_per_token"]
|
||||
> model_group_info.input_cost_per_token
|
||||
):
|
||||
model_group_info.input_cost_per_token = model_info[
|
||||
"input_cost_per_token"
|
||||
]
|
||||
if model_info.get("output_cost_per_token", None) is not None and (
|
||||
model_group_info.output_cost_per_token is None
|
||||
or model_info["output_cost_per_token"]
|
||||
> model_group_info.output_cost_per_token
|
||||
):
|
||||
model_group_info.output_cost_per_token = model_info[
|
||||
"output_cost_per_token"
|
||||
]
|
||||
if (
|
||||
model_info.get("supports_parallel_function_calling", None)
|
||||
is not None
|
||||
and model_info["supports_parallel_function_calling"] is True # type: ignore
|
||||
):
|
||||
model_group_info.supports_parallel_function_calling = True
|
||||
if (
|
||||
model_info.get("supports_vision", None) is not None
|
||||
and model_info["supports_vision"] is True # type: ignore
|
||||
):
|
||||
model_group_info.supports_vision = True
|
||||
if (
|
||||
model_info.get("supports_function_calling", None) is not None
|
||||
and model_info["supports_function_calling"] is True # type: ignore
|
||||
):
|
||||
model_group_info.supports_function_calling = True
|
||||
if (
|
||||
model_info.get("supported_openai_params", None) is not None
|
||||
and model_info["supported_openai_params"] is not None
|
||||
):
|
||||
model_group_info.supported_openai_params = model_info[
|
||||
"supported_openai_params"
|
||||
]
|
||||
if total_rpm is not None:
|
||||
model_group_info.rpm = total_rpm
|
||||
|
||||
## UPDATE WITH TOTAL TPM/RPM FOR MODEL GROUP
|
||||
if total_tpm is not None and model_group_info is not None:
|
||||
model_group_info.tpm = total_tpm
|
||||
|
||||
if total_rpm is not None and model_group_info is not None:
|
||||
model_group_info.rpm = total_rpm
|
||||
## UPDATE WITH CONFIGURABLE CLIENTSIDE AUTH PARAMS FOR MODEL GROUP
|
||||
if configurable_clientside_auth_params is not None:
|
||||
model_group_info.configurable_clientside_auth_params = (
|
||||
configurable_clientside_auth_params
|
||||
)
|
||||
|
||||
return model_group_info
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue