phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) test_tokenizers

Browse source
This commit is contained in:
ishaan-jaff 2024-02-23 10:08:07 -08:00
parent 44b8270bf6
commit ff815e93a8
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/tests/test_token_counter.py
View file

@ -53,7 +53,7 @@ def test_tokenizers():
try: try:
### test the openai, claude, cohere and llama2 tokenizers. ### test the openai, claude, cohere and llama2 tokenizers.
### The tokenizer value should be different for all ### The tokenizer value should be different for all
sample_text = "Hellö World, this is my input string!" sample_text = "Hellö World, this is my input string! My name is ishaan CTO"
# openai tokenizer # openai tokenizer
openai_tokens = token_counter(model="gpt-3.5-turbo", text=sample_text) openai_tokens = token_counter(model="gpt-3.5-turbo", text=sample_text)