From 6e6849f416709db58917b00430c53853dc439773 Mon Sep 17 00:00:00 2001 From: zhourunlai Date: Wed, 8 Jan 2025 15:57:55 +0800 Subject: [PATCH] polish tiktoken encoding_for_model --- litellm/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 6051a87327..60d4bad3d7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1360,10 +1360,7 @@ def openai_token_counter( # noqa: PLR0915 """ print_verbose(f"LiteLLM: Utils - Counting tokens for OpenAI model={model}") try: - if "gpt-4o" in model: - encoding = tiktoken.get_encoding("o200k_base") - else: - encoding = tiktoken.encoding_for_model(model) + encoding = tiktoken.encoding_for_model(model) except KeyError: print_verbose("Warning: model not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base")