phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix use converse for all llama3 models (#5729)

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-19 09:31:52 -07:00 • committed by GitHub
parent 8497e2aa36
commit cd90807807
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 30 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/__init__.py
View file

@ -917,9 +917,11 @@ from .llms.maritalk import MaritTalkConfig
from .llms.bedrock.chat.invoke_handler import ( from .llms.bedrock.chat.invoke_handler import (
AmazonCohereChatConfig, AmazonCohereChatConfig,
AmazonConverseConfig, AmazonConverseConfig,
BEDROCK_CONVERSE_MODELS,
bedrock_tool_name_mappings, bedrock_tool_name_mappings,
) )
from .llms.bedrock.chat.converse_handler import (
BEDROCK_CONVERSE_MODELS,
)
from .llms.bedrock.common_utils import ( from .llms.bedrock.common_utils import (
AmazonTitanConfig, AmazonTitanConfig,
AmazonAI21Config, AmazonAI21Config,

27
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -18,6 +18,33 @@ from ...base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError from ..common_utils import BedrockError
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
BEDROCK_CONVERSE_MODELS = [
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
"anthropic.claude-3-opus-20240229-v1:0",
"us.anthropic.claude-3-opus-20240229-v1:0",
"eu.anthropic.claude-3-opus-20240229-v1:0",
"anthropic.claude-3-sonnet-20240229-v1:0",
"us.anthropic.claude-3-sonnet-20240229-v1:0",
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
"anthropic.claude-3-haiku-20240307-v1:0",
"us.anthropic.claude-3-haiku-20240307-v1:0",
"eu.anthropic.claude-3-haiku-20240307-v1:0",
"anthropic.claude-v2",
"anthropic.claude-v2:1",
"anthropic.claude-v1",
"anthropic.claude-instant-v1",
"ai21.jamba-instruct-v1:0",
"meta.llama3-70b-instruct-v1:0",
"meta.llama3-8b-instruct-v1:0",
"meta.llama3-1-8b-instruct-v1:0",
"meta.llama3-1-70b-instruct-v1:0",
"meta.llama3-1-405b-instruct-v1:0",
"meta.llama3-70b-instruct-v1:0",
"mistral.mistral-large-2407-v1:0",
]
def make_sync_call( def make_sync_call(
client: Optional[HTTPHandler], client: Optional[HTTPHandler],

26
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -67,32 +67,6 @@ from ...prompt_templates.factory import (
from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name
from .converse_transformation import AmazonConverseConfig from .converse_transformation import AmazonConverseConfig
BEDROCK_CONVERSE_MODELS = [
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
"anthropic.claude-3-opus-20240229-v1:0",
"us.anthropic.claude-3-opus-20240229-v1:0",
"eu.anthropic.claude-3-opus-20240229-v1:0",
"anthropic.claude-3-sonnet-20240229-v1:0",
"us.anthropic.claude-3-sonnet-20240229-v1:0",
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
"anthropic.claude-3-haiku-20240307-v1:0",
"us.anthropic.claude-3-haiku-20240307-v1:0",
"eu.anthropic.claude-3-haiku-20240307-v1:0",
"anthropic.claude-v2",
"anthropic.claude-v2:1",
"anthropic.claude-v1",
"anthropic.claude-instant-v1",
"ai21.jamba-instruct-v1:0",
"meta.llama3-1-8b-instruct-v1:0",
"meta.llama3-1-70b-instruct-v1:0",
"meta.llama3-1-405b-instruct-v1:0",
"meta.llama3-70b-instruct-v1:0",
"mistral.mistral-large-2407-v1:0",
]
_response_stream_shape_cache = None _response_stream_shape_cache = None
bedrock_tool_name_mappings: InMemoryCache = InMemoryCache( bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
max_size_in_memory=50, default_ttl=600 max_size_in_memory=50, default_ttl=600