phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 02:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) token_counter - use openai token counter only for chat completion

Browse source
This commit is contained in:
ishaan-jaff 2023-11-13 08:00:27 -08:00
parent e5ec4a92fe
commit c91abc8ad1
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/utils.py
View file

@ -1323,7 +1323,7 @@ def token_counter(model="", text=None, messages: Optional[List] = None):
enc = tokenizer_json["tokenizer"].encode(text)
num_tokens = len(enc.ids)
elif tokenizer_json["type"] == "openai_tokenizer":
if messages is not None:
if model in litellm.open_ai_chat_completion_models and messages != None:
num_tokens = openai_token_counter(messages, model=model)
else:
enc = tokenizer_json["tokenizer"].encode(text)