phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 10:14:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

polish tiktoken encoding_for_model

Browse source
This commit is contained in:
zhourunlai 2025-01-08 15:57:55 +08:00
parent 07c5f136f1
commit 6e6849f416
1 changed files with 1 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/utils.py
View file

@ -1360,10 +1360,7 @@ def openai_token_counter( # noqa: PLR0915
"""
print_verbose(f"LiteLLM: Utils - Counting tokens for OpenAI model={model}")
try:
if "gpt-4o" in model:
encoding = tiktoken.get_encoding("o200k_base")
else:
encoding = tiktoken.encoding_for_model(model)
encoding = tiktoken.encoding_for_model(model)
except KeyError:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
encoding = tiktoken.get_encoding("cl100k_base")