mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 10:44:24 +00:00
(docs) encode docstring
This commit is contained in:
parent
313eee17e3
commit
c61fa70ba0
2 changed files with 11 additions and 1 deletions
|
@ -911,6 +911,16 @@ def _select_tokenizer(model: str):
|
|||
return {"type": "openai_tokenizer", "tokenizer": encoding}
|
||||
|
||||
def encode(model: str, text: str):
|
||||
"""
|
||||
Encodes the given text using the specified model.
|
||||
|
||||
Args:
|
||||
model (str): The name of the model to use for tokenization.
|
||||
text (str): The text to be encoded.
|
||||
|
||||
Returns:
|
||||
enc: The encoded text.
|
||||
"""
|
||||
tokenizer_json = _select_tokenizer(model=model)
|
||||
enc = tokenizer_json["tokenizer"].encode(text)
|
||||
return enc
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue