phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) merge conflict

Browse source
This commit is contained in:
ishaan-jaff 2024-02-15 18:25:19 -08:00
parent cb7f380829
commit ec16b536a1
1 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

13
litellm/utils.py
View file

@ -34,11 +34,14 @@ from dataclasses import (
# import pkg_resources # import pkg_resources
from importlib import resources from importlib import resources
# # filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") try:
# filename = str(resources.files().joinpath("llms/tokenizers")) filename = str(
# os.environ[ resources.files().joinpath("llms/tokenizers") # type: ignore
# "TIKTOKEN_CACHE_DIR" ) # for python 3.8 and 3.12
# ] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 except:
filename = str(
resources.files(litellm).joinpath("llms/tokenizers") # for python 3.10
) # for python 3.10+
encoding = tiktoken.get_encoding("cl100k_base") encoding = tiktoken.get_encoding("cl100k_base")
import importlib.metadata import importlib.metadata
from ._logging import verbose_logger from ._logging import verbose_logger