mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 18:54:30 +00:00
Litellm dev 12 28 2024 p1 (#7463)
* refactor(utils.py): migrate amazon titan config to base config * refactor(utils.py): refactor bedrock meta invoke model translation to use base config * refactor(utils.py): move bedrock ai21 to base config * refactor(utils.py): move bedrock cohere to base config * refactor(utils.py): move bedrock mistral to use base config * refactor(utils.py): move all provider optional param translations to using a config * docs(clientside_auth.md): clarify how to pass vertex region to litellm proxy * fix(utils.py): handle scenario where custom llm provider is none / empty * fix: fix get config * test(test_otel_load_tests.py): widen perf margin * fix(utils.py): fix get provider config check to handle custom llm's * fix(utils.py): fix check
This commit is contained in:
Krish Dholakia
GitHub
parent
ec7fcc982d
commit
31ace870a2
11 changed files with 753 additions and 446 deletions
|
|
@ -3,27 +3,26 @@ Common utilities used across bedrock chat/embedding/image generation
|
|||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import types
|
||||
from enum import Enum
|
||||
from typing import List, Optional, Union
|
||||
from typing import Any, List, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.chat.transformation import (
|
||||
BaseConfig,
|
||||
BaseLLMException,
|
||||
LiteLLMLoggingObj,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
||||
class BedrockError(Exception):
|
||||
def __init__(self, status_code, message):
|
||||
self.status_code = status_code
|
||||
self.message = message
|
||||
self.request = httpx.Request(
|
||||
method="POST", url="https://us-west-2.console.aws.amazon.com/bedrock"
|
||||
)
|
||||
self.response = httpx.Response(status_code=status_code, request=self.request)
|
||||
super().__init__(
|
||||
self.message
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
class BedrockError(BaseLLMException):
|
||||
pass
|
||||
|
||||
|
||||
class AmazonBedrockGlobalConfig:
|
||||
|
|
@ -65,7 +64,64 @@ class AmazonBedrockGlobalConfig:
|
|||
]
|
||||
|
||||
|
||||
class AmazonTitanConfig:
|
||||
class AmazonInvokeMixin:
|
||||
"""
|
||||
Base class for bedrock models going through invoke_handler.py
|
||||
"""
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
return BedrockError(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
raise NotImplementedError(
|
||||
"transform_request not implemented for config. Done in invoke_handler.py"
|
||||
)
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
raise NotImplementedError(
|
||||
"transform_response not implemented for config. Done in invoke_handler.py"
|
||||
)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
) -> dict:
|
||||
raise NotImplementedError(
|
||||
"validate_environment not implemented for config. Done in invoke_handler.py"
|
||||
)
|
||||
|
||||
|
||||
class AmazonTitanConfig(AmazonInvokeMixin, BaseConfig):
|
||||
"""
|
||||
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-text-express-v1
|
||||
|
||||
|
|
@ -100,6 +156,7 @@ class AmazonTitanConfig:
|
|||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
if not k.startswith("__")
|
||||
and not k.startswith("_abc")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
|
|
@ -112,6 +169,62 @@ class AmazonTitanConfig:
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def _map_and_modify_arg(
|
||||
self,
|
||||
supported_params: dict,
|
||||
provider: str,
|
||||
model: str,
|
||||
stop: Union[List[str], str],
|
||||
):
|
||||
"""
|
||||
filter params to fit the required provider format, drop those that don't fit if user sets `litellm.drop_params = True`.
|
||||
"""
|
||||
filtered_stop = None
|
||||
if "stop" in supported_params and litellm.drop_params:
|
||||
if provider == "bedrock" and "amazon" in model:
|
||||
filtered_stop = []
|
||||
if isinstance(stop, list):
|
||||
for s in stop:
|
||||
if re.match(r"^(\|+|User:)$", s):
|
||||
filtered_stop.append(s)
|
||||
if filtered_stop is not None:
|
||||
supported_params["stop"] = filtered_stop
|
||||
|
||||
return supported_params
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
return [
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"stream",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for k, v in non_default_params.items():
|
||||
if k == "max_tokens" or k == "max_completion_tokens":
|
||||
optional_params["maxTokenCount"] = v
|
||||
if k == "temperature":
|
||||
optional_params["temperature"] = v
|
||||
if k == "stop":
|
||||
filtered_stop = self._map_and_modify_arg(
|
||||
{"stop": v}, provider="bedrock", model=model, stop=v
|
||||
)
|
||||
optional_params["stopSequences"] = filtered_stop["stop"]
|
||||
if k == "top_p":
|
||||
optional_params["topP"] = v
|
||||
if k == "stream":
|
||||
optional_params["stream"] = v
|
||||
return optional_params
|
||||
|
||||
|
||||
class AmazonAnthropicClaude3Config:
|
||||
"""
|
||||
|
|
@ -276,7 +389,7 @@ class AmazonAnthropicConfig:
|
|||
return optional_params
|
||||
|
||||
|
||||
class AmazonCohereConfig:
|
||||
class AmazonCohereConfig(AmazonInvokeMixin, BaseConfig):
|
||||
"""
|
||||
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command
|
||||
|
||||
|
|
@ -308,6 +421,7 @@ class AmazonCohereConfig:
|
|||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
if not k.startswith("__")
|
||||
and not k.startswith("_abc")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
|
|
@ -320,8 +434,31 @@ class AmazonCohereConfig:
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
return [
|
||||
"max_tokens",
|
||||
"temperature",
|
||||
"stream",
|
||||
]
|
||||
|
||||
class AmazonAI21Config:
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for k, v in non_default_params.items():
|
||||
if k == "stream":
|
||||
optional_params["stream"] = v
|
||||
if k == "temperature":
|
||||
optional_params["temperature"] = v
|
||||
if k == "max_tokens":
|
||||
optional_params["max_tokens"] = v
|
||||
return optional_params
|
||||
|
||||
|
||||
class AmazonAI21Config(AmazonInvokeMixin, BaseConfig):
|
||||
"""
|
||||
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra
|
||||
|
||||
|
|
@ -371,6 +508,7 @@ class AmazonAI21Config:
|
|||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
if not k.startswith("__")
|
||||
and not k.startswith("_abc")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
|
|
@ -383,13 +521,39 @@ class AmazonAI21Config:
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List:
|
||||
return [
|
||||
"max_tokens",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"stream",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for k, v in non_default_params.items():
|
||||
if k == "max_tokens":
|
||||
optional_params["maxTokens"] = v
|
||||
if k == "temperature":
|
||||
optional_params["temperature"] = v
|
||||
if k == "top_p":
|
||||
optional_params["topP"] = v
|
||||
if k == "stream":
|
||||
optional_params["stream"] = v
|
||||
return optional_params
|
||||
|
||||
|
||||
class AnthropicConstants(Enum):
|
||||
HUMAN_PROMPT = "\n\nHuman: "
|
||||
AI_PROMPT = "\n\nAssistant: "
|
||||
|
||||
|
||||
class AmazonLlamaConfig:
|
||||
class AmazonLlamaConfig(AmazonInvokeMixin, BaseConfig):
|
||||
"""
|
||||
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=meta.llama2-13b-chat-v1
|
||||
|
||||
|
|
@ -421,6 +585,7 @@ class AmazonLlamaConfig:
|
|||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
if not k.startswith("__")
|
||||
and not k.startswith("_abc")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
|
|
@ -433,8 +598,34 @@ class AmazonLlamaConfig:
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List:
|
||||
return [
|
||||
"max_tokens",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"stream",
|
||||
]
|
||||
|
||||
class AmazonMistralConfig:
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for k, v in non_default_params.items():
|
||||
if k == "max_tokens":
|
||||
optional_params["max_gen_len"] = v
|
||||
if k == "temperature":
|
||||
optional_params["temperature"] = v
|
||||
if k == "top_p":
|
||||
optional_params["top_p"] = v
|
||||
if k == "stream":
|
||||
optional_params["stream"] = v
|
||||
return optional_params
|
||||
|
||||
|
||||
class AmazonMistralConfig(AmazonInvokeMixin, BaseConfig):
|
||||
"""
|
||||
Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral.html
|
||||
Supported Params for the Amazon / Mistral models:
|
||||
|
|
@ -471,6 +662,7 @@ class AmazonMistralConfig:
|
|||
k: v
|
||||
for k, v in cls.__dict__.items()
|
||||
if not k.startswith("__")
|
||||
and not k.startswith("_abc")
|
||||
and not isinstance(
|
||||
v,
|
||||
(
|
||||
|
|
@ -483,6 +675,29 @@ class AmazonMistralConfig:
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
return ["max_tokens", "temperature", "top_p", "stop", "stream"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
for k, v in non_default_params.items():
|
||||
if k == "max_tokens":
|
||||
optional_params["max_tokens"] = v
|
||||
if k == "temperature":
|
||||
optional_params["temperature"] = v
|
||||
if k == "top_p":
|
||||
optional_params["top_p"] = v
|
||||
if k == "stop":
|
||||
optional_params["stop"] = v
|
||||
if k == "stream":
|
||||
optional_params["stream"] = v
|
||||
return optional_params
|
||||
|
||||
|
||||
def add_custom_header(headers):
|
||||
"""Closure to capture the headers and add them."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue