From 7d7b59ff78a44c9375167dabb79bb16b56a17c91 Mon Sep 17 00:00:00 2001 From: nkvch Date: Mon, 6 May 2024 16:59:13 +0200 Subject: [PATCH 1/3] * feat(factory.py): add support for merging consecutive messages of one role when separated with empty message of another role --- litellm/llms/prompt_templates/factory.py | 62 ++++++++++++++---------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 082030368..6e8589d58 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1,27 +1,23 @@ -from enum import Enum -import requests, traceback -import json, re, xml.etree.ElementTree as ET -from jinja2 import Template, exceptions, meta, BaseLoader -from jinja2.sandbox import ImmutableSandboxedEnvironment -from typing import ( - Any, - List, - Mapping, - MutableMapping, - Optional, - Sequence, -) -import litellm -from litellm.types.completion import ( - ChatCompletionUserMessageParam, - ChatCompletionSystemMessageParam, - ChatCompletionMessageParam, - ChatCompletionFunctionMessageParam, - ChatCompletionMessageToolCallParam, - ChatCompletionToolMessageParam, -) -from litellm.types.llms.anthropic import * +import json +import re +import traceback import uuid +import xml.etree.ElementTree as ET +from enum import Enum +from typing import Any, List, Mapping, MutableMapping, Optional, Sequence + +import requests +from jinja2 import BaseLoader, Template, exceptions, meta +from jinja2.sandbox import ImmutableSandboxedEnvironment + +import litellm +from litellm.types.completion import (ChatCompletionFunctionMessageParam, + ChatCompletionMessageParam, + ChatCompletionMessageToolCallParam, + ChatCompletionSystemMessageParam, + ChatCompletionToolMessageParam, + ChatCompletionUserMessageParam) +from litellm.types.llms.anthropic import * def default_pt(messages): @@ -603,9 +599,10 @@ def construct_tool_use_system_prompt( def convert_url_to_base64(url): - import requests import base64 + import requests + for _ in range(3): try: response = requests.get(url) @@ -984,6 +981,7 @@ def anthropic_messages_pt(messages: list): new_messages = [] msg_i = 0 tool_use_param = False + merge_with_previous = False while msg_i < len(messages): user_content = [] init_msg_i = msg_i @@ -1016,7 +1014,13 @@ def anthropic_messages_pt(messages: list): msg_i += 1 if user_content: - new_messages.append({"role": "user", "content": user_content}) + if merge_with_previous: + new_messages[-1]["content"].extend(user_content) + merge_with_previous = False + else: + new_messages.append({"role": "user", "content": user_content}) + else: + merge_with_previous = True assistant_content = [] ## MERGE CONSECUTIVE ASSISTANT CONTENT ## @@ -1044,7 +1048,13 @@ def anthropic_messages_pt(messages: list): msg_i += 1 if assistant_content: - new_messages.append({"role": "assistant", "content": assistant_content}) + if merge_with_previous: + new_messages[-1]["content"].extend(assistant_content) + merge_with_previous = False + else: + new_messages.append({"role": "assistant", "content": assistant_content}) + else: + merge_with_previous = True if msg_i == init_msg_i: # prevent infinite loops raise Exception( From 389530efb4aca53048d84d61766b487d8ed1b192 Mon Sep 17 00:00:00 2001 From: nkvch Date: Tue, 7 May 2024 12:45:16 +0200 Subject: [PATCH 2/3] * chore(.gitignore): add 'venv' to the list of ignored files/directories * fix(test_completion.py): fix import order and remove unused imports * feat(test_completion.py): add test for empty assistant message in completion_claude_3_empty_message() --- .gitignore | 1 + litellm/tests/test_completion.py | 38 ++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index abc4ecb0c..4f3f65b93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv +venv .env litellm_uuid.txt __pycache__/ diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 3d0907814..54f040cfa 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1,17 +1,21 @@ -import sys, os +import os +import sys import traceback + from dotenv import load_dotenv load_dotenv() -import os, io +import io +import os sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the, system path import pytest + import litellm -from litellm import embedding, completion, completion_cost, Timeout -from litellm import RateLimitError +from litellm import (RateLimitError, Timeout, completion, completion_cost, + embedding) from litellm.llms.prompt_templates.factory import anthropic_messages_pt # litellm.num_retries=3 @@ -163,6 +167,32 @@ def test_completion_claude_3(): pytest.fail(f"Error occurred: {e}") +def test_completion_claude_3_empty_message(): + litellm.set_verbose = True + messages = [{'role': 'user', 'content': 'please create a logo for a modern AI app. create in SVG format'}, + {'role': 'assistant', 'content': "To create a logo for a modern AI app in SVG format, I'll use the DALL-E 3 Image Generator."}, + {'role': 'user', 'content': 'output SVG'}, + {'role': 'assistant', 'content': 'To generate a logo for a modern AI app in SVG format using DALL-E 3, I need to:\n1. Craft a detailed prompt describing the desired logo style and elements\n2. Specify the image size (SVG is vector-based, so size is less relevant)\n3. Call the generate_image function with the prompt and size\n4. Display the generated SVG logo using the provided syntax\nThe prompt should include keywords related to AI, modern design, and SVG format. Some key elements to incorporate could be a brain symbol, circuit lines, or a robot icon, using a minimalist style and a blue color scheme often associated with technology and intelligence.', + 'tool_calls': [ + {'id': 'toolu_01KEUtRVySSeMrf3g7rCA12E', 'type': 'function', 'function': {'name': 'python_tool', 'arguments': '{"code": "...python code..."}'}} + ]}, + {'role': 'tool', 'content': '...python output...', 'tool_call_id': 'toolu_01KEUtRVySSeMrf3g7rCA12E'}, + {'role': 'assistant', 'content': ''}, # empty message appended by model after tool call response + {'role': 'user', 'content': 'write SVG source youself!'}, + ] + + try: + response = completion( + model="anthropic/claude-3-opus-20240229", + messages=messages, + stream=True, + tools=[{'type': 'function', 'function': {'name': 'python_tool', 'description': 'Execute code', 'parameters': {'type': 'object', 'properties': {'headline': {'description': 'Must have. Title of this tool call (maximum 15 characters).', 'type': 'string'}, 'code': {'description': 'Python code to execute.', 'type': 'string'}}, 'required': ['code', 'headline']}}}] + ) + print(response) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + def test_completion_claude_3_function_call(): litellm.set_verbose = True tools = [ From 06d1aec49ef6f30601ebf6043b8af8de071d005d Mon Sep 17 00:00:00 2001 From: nkvch Date: Tue, 7 May 2024 13:42:42 +0200 Subject: [PATCH 3/3] * fix(factory.py): fix conditional statement for merging messages with previous ones --- litellm/llms/prompt_templates/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 6e8589d58..bce472ea0 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1019,7 +1019,7 @@ def anthropic_messages_pt(messages: list): merge_with_previous = False else: new_messages.append({"role": "user", "content": user_content}) - else: + elif msg_i > 0: merge_with_previous = True assistant_content = [] @@ -1053,7 +1053,7 @@ def anthropic_messages_pt(messages: list): merge_with_previous = False else: new_messages.append({"role": "assistant", "content": assistant_content}) - else: + elif msg_i > 0: merge_with_previous = True if msg_i == init_msg_i: # prevent infinite loops