phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 18:24:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 81d265b0f0 into b82af5b826

Browse source
This commit is contained in:
Andrew Bradley 2025-04-24 01:04:56 -07:00 • committed by GitHub
commit 2f31779a51
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
litellm/utils.py
View file

@ -1566,9 +1566,12 @@ def _return_huggingface_tokenizer(model: str) -> Optional[SelectTokenizerRespons
)
return {"type": "huggingface_tokenizer", "tokenizer": cohere_tokenizer}
# anthropic
elif model in litellm.anthropic_models and "claude-3" not in model:
claude_tokenizer = Tokenizer.from_str(claude_json_str)
return {"type": "huggingface_tokenizer", "tokenizer": claude_tokenizer}
elif model in litellm.anthropic_models:
if "claude-3" not in model:
claude_tokenizer = Tokenizer.from_str(claude_json_str)
return {"type": "huggingface_tokenizer", "tokenizer": claude_tokenizer}
else:
return {"type": "openai_tokenizer", "tokenizer": encoding}
# llama2
elif "llama-2" in model.lower() or "replicate" in model.lower():
tokenizer = Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")