phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) token_counter - use openai token counter only for chat completion

Browse source
This commit is contained in:
ishaan-jaff 2023-11-13 08:00:27 -08:00
parent e5ec4a92fe
commit c91abc8ad1
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/utils.py
View file

@ -1323,7 +1323,7 @@ def token_counter(model="", text=None, messages: Optional[List] = None):
enc = tokenizer_json["tokenizer"].encode(text) enc = tokenizer_json["tokenizer"].encode(text)
num_tokens = len(enc.ids) num_tokens = len(enc.ids)
elif tokenizer_json["type"] == "openai_tokenizer": elif tokenizer_json["type"] == "openai_tokenizer":
if messages is not None: if model in litellm.open_ai_chat_completion_models and messages != None:
num_tokens = openai_token_counter(messages, model=model) num_tokens = openai_token_counter(messages, model=model)
else: else:
enc = tokenizer_json["tokenizer"].encode(text) enc = tokenizer_json["tokenizer"].encode(text)