phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

include methods in init import, add test, fix encode/decode param ordering

Browse source
This commit is contained in:
Christian Privitelli 2024-05-02 15:49:22 +10:00
parent 3449a5e446
commit 2d43153efa
5 changed files with 19 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/utils.py
View file

@ -3696,7 +3696,7 @@ def _select_tokenizer(model: str):
return {"type": "openai_tokenizer", "tokenizer": encoding}
def encode(model: str, custom_tokenizer: Optional[dict] = None, text: str):
def encode(model="", text="", custom_tokenizer: Optional[dict] = None):
"""
Encodes the given text using the specified model.
@ -3713,7 +3713,7 @@ def encode(model: str, custom_tokenizer: Optional[dict] = None, text: str):
return enc
def decode(model: str, custom_tokenizer: Optional[dict] = None, tokens: List[int]):
def decode(model="", tokens: List[int] = [], custom_tokenizer: Optional[dict] = None):
tokenizer_json = custom_tokenizer or _select_tokenizer(model=model)
dec = tokenizer_json["tokenizer"].decode(tokens)
return dec