diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index e093325829..05bf78a6a9 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -6,12 +6,12 @@ try: # New and recommended way to access resources from importlib import resources - filename = str(resources.files(litellm).joinpath("llms/tokenizers")) + filename = str(resources.files(litellm).joinpath("litellm_core_utils/tokenizers")) except (ImportError, AttributeError): # Old way to access resources, which setuptools deprecated some time ago import pkg_resources # type: ignore - filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") + filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers") os.environ["TIKTOKEN_CACHE_DIR"] = os.getenv( "CUSTOM_TIKTOKEN_CACHE_DIR", filename diff --git a/litellm/llms/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 b/litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 similarity index 100% rename from litellm/llms/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 rename to litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 diff --git a/litellm/llms/tokenizers/__init__.py b/litellm/litellm_core_utils/tokenizers/__init__.py similarity index 100% rename from litellm/llms/tokenizers/__init__.py rename to litellm/litellm_core_utils/tokenizers/__init__.py diff --git a/litellm/llms/tokenizers/anthropic_tokenizer.json b/litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json similarity index 100% rename from litellm/llms/tokenizers/anthropic_tokenizer.json rename to litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json diff --git a/litellm/llms/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15 b/litellm/litellm_core_utils/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15 similarity index 100% rename from litellm/llms/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15 rename to litellm/litellm_core_utils/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15 diff --git a/litellm/llms/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790 b/litellm/litellm_core_utils/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790 similarity index 100% rename from litellm/llms/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790 rename to litellm/litellm_core_utils/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790 diff --git a/litellm/utils.py b/litellm/utils.py index 05af5d0252..2d99ac5188 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -135,7 +135,7 @@ from litellm.types.utils import ( Usage, ) -with resources.open_text("litellm.llms.tokenizers", "anthropic_tokenizer.json") as f: +with resources.open_text("litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json") as f: json_data = json.load(f) # Convert to str (if necessary) claude_json_str = json.dumps(json_data) diff --git a/tests/local_testing/test_secret_manager.py b/tests/local_testing/test_secret_manager.py index b2c01f5bbd..f4fb1b450a 100644 --- a/tests/local_testing/test_secret_manager.py +++ b/tests/local_testing/test_secret_manager.py @@ -231,7 +231,7 @@ def test_google_secret_manager_read_in_memory(): from litellm.secret_managers.google_secret_manager import GoogleSecretManager - # load_vertex_ai_credentials() + load_vertex_ai_credentials() os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "adroit-crow-413218" secret_manager = GoogleSecretManager() secret_manager.cache.cache_dict["UNIQUE_KEY"] = None