From ec16b536a1f7590cface53a63995d38346a96aa8 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 18:25:19 -0800 Subject: [PATCH] (fix) merge conflict --- litellm/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index c05e2524a..6c286c87d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -34,11 +34,14 @@ from dataclasses import ( # import pkg_resources from importlib import resources -# # filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") -# filename = str(resources.files().joinpath("llms/tokenizers")) -# os.environ[ -# "TIKTOKEN_CACHE_DIR" -# ] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 +try: + filename = str( + resources.files().joinpath("llms/tokenizers") # type: ignore + ) # for python 3.8 and 3.12 +except: + filename = str( + resources.files(litellm).joinpath("llms/tokenizers") # for python 3.10 + ) # for python 3.10+ encoding = tiktoken.get_encoding("cl100k_base") import importlib.metadata from ._logging import verbose_logger