From a18844b2309d67bce975302cfff41e8f3d1fb0e6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 08:28:31 -0700 Subject: [PATCH] fix(utils.py): use llama tokenizer for replicate models --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index e1d80c2d2..45bfba303 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3668,7 +3668,7 @@ def _select_tokenizer(model: str): tokenizer = Tokenizer.from_str(json_str) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} # llama2 - elif "llama-2" in model.lower(): + elif "llama-2" in model.lower() or "replicate" in model.lower(): tokenizer = Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} # default - tiktoken