phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(utils.py): fix local tiktoken package import

Browse source
This commit is contained in:
Krrish Dholakia 2024-02-15 16:18:54 -08:00
parent 61125cf982
commit 3a5d40f4b9
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/utils.py
View file

@ -29,11 +29,12 @@ from dataclasses import (
dataclass,
field,
) # for storing API inputs, outputs, and metadata
#import pkg_resources
# import pkg_resources
from importlib import resources
# filename = pkg_resources.resource_filename(__name__, "llms/tokenizers")
filename = str(resources.files().joinpath("llms/tokenizers"))
filename = str(resources.files(litellm).joinpath("llms/tokenizers"))
os.environ[
"TIKTOKEN_CACHE_DIR"
] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071