From c61fa70ba031022cfda80b24a6028d69c6fb34e5 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 30 Oct 2023 14:10:29 -0700 Subject: [PATCH] (docs) encode docstring --- docs/my-website/docs/completion/token_usage.md | 2 +- litellm/utils.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/completion/token_usage.md b/docs/my-website/docs/completion/token_usage.md index 8b1140bad..56274682f 100644 --- a/docs/my-website/docs/completion/token_usage.md +++ b/docs/my-website/docs/completion/token_usage.md @@ -39,7 +39,7 @@ def test_encoding_and_decoding(): # openai tokenizer openai_tokens = token_counter(model="gpt-3.5-turbo", text=sample_text) - openai_text = decode(model="gpt-3.5-turbo", tokens=openai_tokens) + openai_text = encode(model="gpt-3.5-turbo", tokens=openai_tokens) assert openai_text == sample_text except: diff --git a/litellm/utils.py b/litellm/utils.py index ebf06caff..e33e494bc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -911,6 +911,16 @@ def _select_tokenizer(model: str): return {"type": "openai_tokenizer", "tokenizer": encoding} def encode(model: str, text: str): + """ + Encodes the given text using the specified model. + + Args: + model (str): The name of the model to use for tokenization. + text (str): The text to be encoded. + + Returns: + enc: The encoded text. + """ tokenizer_json = _select_tokenizer(model=model) enc = tokenizer_json["tokenizer"].encode(text) return enc