diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index b49cae5f3..e776bee50 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -2,7 +2,7 @@ from enum import Enum import requests, traceback import json, re, xml.etree.ElementTree as ET from jinja2 import Template, exceptions, Environment, meta -from typing import Optional, Any, List +from typing import Optional, Any import imghdr, base64 @@ -481,6 +481,34 @@ def construct_tool_use_system_prompt( return tool_use_system_prompt +def convert_url_to_base64(url): + import requests + import base64 + + response = requests.get(url) + if response.status_code == 200: + image_bytes = response.content + base64_image = base64.b64encode(image_bytes).decode("utf-8") + + img_type = url.split(".")[-1].lower() + if img_type == "jpg" or img_type == "jpeg": + img_type = "image/jpeg" + elif img_type == "png": + img_type = "image/png" + elif img_type == "gif": + img_type = "image/gif" + elif img_type == "webp": + img_type = "image/webp" + else: + raise Exception( + f"Error: Unsupported image format. Format={img_type}. Supported types = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']" + ) + + return f"data:{img_type};base64,{base64_image}" + else: + raise Exception(f"Error: Unable to fetch image from URL. url={url}") + + def convert_to_anthropic_image_obj(openai_image_url: str): """ Input: @@ -493,17 +521,24 @@ def convert_to_anthropic_image_obj(openai_image_url: str): "data": {base64_image}, } """ - # Extract the base64 image data - base64_data = openai_image_url.split("data:image/")[1].split(";base64,")[1] + try: + if openai_image_url.startswith("http"): + openai_image_url = convert_url_to_base64(url=openai_image_url) + # Extract the base64 image data + base64_data = openai_image_url.split("data:image/")[1].split(";base64,")[1] - # Infer image format from the URL - image_format = openai_image_url.split("data:image/")[1].split(";base64,")[0] + # Infer image format from the URL + image_format = openai_image_url.split("data:image/")[1].split(";base64,")[0] - return { - "type": "base64", - "media_type": f"image/{image_format}", - "data": base64_data, - } + return { + "type": "base64", + "media_type": f"image/{image_format}", + "data": base64_data, + } + except Exception as e: + raise Exception( + """Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{base64_image}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp'] """ + ) def anthropic_messages_pt(messages: list): @@ -586,7 +621,7 @@ def anthropic_messages_pt(messages: list): return new_messages -def extract_between_tags(tag: str, string: str, strip: bool = False) -> List[str]: +def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: ext_list = re.findall(f"<{tag}>(.+?)", string, re.DOTALL) if strip: ext_list = [e.strip() for e in ext_list] diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 1423c7c43..af00275d3 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -151,8 +151,6 @@ def test_completion_claude_3_function_call(): assert isinstance( response.choices[0].message.tool_calls[0].function.arguments, str ) - except litellm.ServiceUnavailableError as e: - pass except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -221,6 +219,55 @@ def test_completion_claude_3_base64(): pytest.fail(f"An exception occurred - {str(e)}") +def test_completion_claude_3_function_plus_image(): + litellm.set_verbose = True + + image_content = [ + {"type": "text", "text": "What’s in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + }, + }, + ] + image_message = {"role": "user", "content": image_content} + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "text", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + + tool_choice = {"type": "function", "function": {"name": "get_current_weather"}} + messages = [{"role": "user", "content": "What's the weather like in Boston today?"}] + + response = completion( + model="claude-3-sonnet-20240229", + messages=[image_message], + tool_choice=tool_choice, + tools=tools, + stream=False, + ) + + print(response) + + def test_completion_mistral_api(): try: litellm.set_verbose = True @@ -1406,9 +1453,9 @@ def test_completion_replicate_vicuna(): def test_replicate_custom_prompt_dict(): litellm.set_verbose = True - model_name = "replicate/mistralai/mixtral-8x7b-instruct-v0.1" + model_name = "replicate/meta/llama-2-7b-chat:13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0" litellm.register_prompt_template( - model="replicate/mistralai/mixtral-8x7b-instruct-v0.1", + model="replicate/meta/llama-2-7b-chat:13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0", initial_prompt_value="You are a good assistant", # [OPTIONAL] roles={ "system": {