From 3055d9b81c2fc07f09a72681b74dd4fc3b438a06 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Tue, 10 Dec 2024 23:50:15 -0800
Subject: [PATCH] Code Quality Improvement - remove `tokenizers/` from /llms
 (#7163)

* move tokenizers out of /llms

* use updated tokenizers location

* fix test_google_secret_manager_read_in_memory
---
 litellm/litellm_core_utils/default_encoding.py                | 4 ++--
 .../tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4       | 0
 litellm/{llms => litellm_core_utils}/tokenizers/__init__.py   | 0
 .../tokenizers/anthropic_tokenizer.json                       | 0
 .../tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15       | 0
 .../tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790       | 0
 litellm/utils.py                                              | 2 +-
 tests/local_testing/test_secret_manager.py                    | 2 +-
 8 files changed, 4 insertions(+), 4 deletions(-)
 rename litellm/{llms => litellm_core_utils}/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 (100%)
 rename litellm/{llms => litellm_core_utils}/tokenizers/__init__.py (100%)
 rename litellm/{llms => litellm_core_utils}/tokenizers/anthropic_tokenizer.json (100%)
 rename litellm/{llms => litellm_core_utils}/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15 (100%)
 rename litellm/{llms => litellm_core_utils}/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790 (100%)

diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py
index e093325829..05bf78a6a9 100644
--- a/litellm/litellm_core_utils/default_encoding.py
+++ b/litellm/litellm_core_utils/default_encoding.py
@@ -6,12 +6,12 @@ try:
     # New and recommended way to access resources
     from importlib import resources
 
-    filename = str(resources.files(litellm).joinpath("llms/tokenizers"))
+    filename = str(resources.files(litellm).joinpath("litellm_core_utils/tokenizers"))
 except (ImportError, AttributeError):
     # Old way to access resources, which setuptools deprecated some time ago
     import pkg_resources  # type: ignore
 
-    filename = pkg_resources.resource_filename(__name__, "llms/tokenizers")
+    filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers")
 
 os.environ["TIKTOKEN_CACHE_DIR"] = os.getenv(
     "CUSTOM_TIKTOKEN_CACHE_DIR", filename
diff --git a/litellm/llms/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 b/litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
similarity index 100%
rename from litellm/llms/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
rename to litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
diff --git a/litellm/llms/tokenizers/__init__.py b/litellm/litellm_core_utils/tokenizers/__init__.py
similarity index 100%
rename from litellm/llms/tokenizers/__init__.py
rename to litellm/litellm_core_utils/tokenizers/__init__.py
diff --git a/litellm/llms/tokenizers/anthropic_tokenizer.json b/litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json
similarity index 100%
rename from litellm/llms/tokenizers/anthropic_tokenizer.json
rename to litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json
diff --git a/litellm/llms/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15 b/litellm/litellm_core_utils/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15
similarity index 100%
rename from litellm/llms/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15
rename to litellm/litellm_core_utils/tokenizers/ec7223a39ce59f226a68acc30dc1af2788490e15
diff --git a/litellm/llms/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790 b/litellm/litellm_core_utils/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790
similarity index 100%
rename from litellm/llms/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790
rename to litellm/litellm_core_utils/tokenizers/fb374d419588a4632f3f557e76b4b70aebbca790
diff --git a/litellm/utils.py b/litellm/utils.py
index 05af5d0252..2d99ac5188 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -135,7 +135,7 @@ from litellm.types.utils import (
     Usage,
 )
 
-with resources.open_text("litellm.llms.tokenizers", "anthropic_tokenizer.json") as f:
+with resources.open_text("litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json") as f:
     json_data = json.load(f)
 # Convert to str (if necessary)
 claude_json_str = json.dumps(json_data)
diff --git a/tests/local_testing/test_secret_manager.py b/tests/local_testing/test_secret_manager.py
index b2c01f5bbd..f4fb1b450a 100644
--- a/tests/local_testing/test_secret_manager.py
+++ b/tests/local_testing/test_secret_manager.py
@@ -231,7 +231,7 @@ def test_google_secret_manager_read_in_memory():
 
     from litellm.secret_managers.google_secret_manager import GoogleSecretManager
 
-    # load_vertex_ai_credentials()
+    load_vertex_ai_credentials()
     os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "adroit-crow-413218"
     secret_manager = GoogleSecretManager()
     secret_manager.cache.cache_dict["UNIQUE_KEY"] = None