From aa36bd2784101cb1bca0f4c83ff41eda62e2d318 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 30 Nov 2023 18:51:51 -0800 Subject: [PATCH] fix(utils.py): expand openai_token_counter selection --- litellm/utils.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index a3908e71c..6fb68d481 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1572,21 +1572,12 @@ def openai_token_counter(messages: Optional[list]=None, model="gpt-3.5-turbo-061 except KeyError: print_verbose("Warning: model not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base") - if model in { - "gpt-3.5-turbo", - "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-16k-0613", - "gpt-4", - "gpt-4-0314", - "gpt-4-32k-0314", - "gpt-4-0613", - "gpt-4-32k-0613", - }: - tokens_per_message = 3 - tokens_per_name = 1 - elif model == "gpt-3.5-turbo-0301": + if model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n tokens_per_name = -1 # if there's a name, the role is omitted + elif model in litellm.open_ai_chat_completion_models: + tokens_per_message = 3 + tokens_per_name = 1 else: raise NotImplementedError( f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."""