phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update utils.py: switched from pkg_resources to importlib.resources for filename stored in the os/environ(TIKTOKEN_CACHE_DIR)

Browse source
This commit is contained in:
Alexandre Sorokine 2024-02-12 16:28:38 -05:00
parent 1ba56d8ff0
commit 36360c62ac
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
litellm/utils.py
View file

@ -29,9 +29,11 @@ from dataclasses import (
dataclass,
field,
) # for storing API inputs, outputs, and metadata
import pkg_resources
#import pkg_resources
from importlib import resources
filename = pkg_resources.resource_filename(__name__, "llms/tokenizers")
# filename = pkg_resources.resource_filename(__name__, "llms/tokenizers")
filename = str(resources.files().joinpath("llms/tokenizers"))
os.environ[
"TIKTOKEN_CACHE_DIR"
] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071